Adopt PromptKit v0.23.0 conversations and execution accounting

This commit is contained in:
2026-11-15 01:25:55 +00:00
parent d28b560265
commit d98d419b36
25 changed files with 689 additions and 924 deletions

View File

@@ -20,15 +20,16 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
}
type executionCall struct {
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
}
type executionEvent struct {
@@ -72,7 +73,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
value := call.Finished
finished = &value
}
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
}
for _, event := range snapshot.Events {
var generation, attempt *uint64
@@ -87,7 +88,8 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
}
return struct {
Calls []executionCall `json:"calls"`
Events []executionEvent `json:"events"`
}{calls, events}
Summary llm.ExecutionSummary `json:"summary"`
Calls []executionCall `json:"calls"`
Events []executionEvent `json:"events"`
}{llm.Summarize(snapshot), calls, events}
}

View File

@@ -19,7 +19,7 @@ func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
},
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
Events: []llm.ExecutionObservation{{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
}
raw, err := json.Marshal(BuildLLMExecution(snapshot))
if err != nil {
@@ -46,7 +46,7 @@ func BenchmarkExecutionReports(b *testing.B) {
snapshot := llm.ObservationSnapshot{}
for i := 0; i < n; i++ {
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{RunID: fmt.Sprint(i), CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
}
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
@@ -64,9 +64,9 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
},
Events: []llm.ExecutionObservation{
{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
{CallID: 3, RunID: "run-3", Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 2, RunID: "run-2", Kind: string(pk.ExecutionFinished), Attempts: 1},
},
}
raw, err := json.Marshal(BuildLLMExecution(snapshot))