Adopt PromptKit v0.23.0 conversations and execution accounting
This commit is contained in:
@@ -20,15 +20,16 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type executionEvent struct {
|
||||
@@ -72,7 +73,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
@@ -87,7 +88,8 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
||||
}
|
||||
return struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{calls, events}
|
||||
Summary llm.ExecutionSummary `json:"summary"`
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{llm.Summarize(snapshot), calls, events}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
|
||||
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
if err != nil {
|
||||
@@ -46,7 +46,7 @@ func BenchmarkExecutionReports(b *testing.B) {
|
||||
snapshot := llm.ObservationSnapshot{}
|
||||
for i := 0; i < n; i++ {
|
||||
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{RunID: fmt.Sprint(i), CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
}
|
||||
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -64,9 +64,9 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{
|
||||
{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
|
||||
{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
|
||||
{CallID: 3, RunID: "run-3", Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 2, RunID: "run-2", Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
|
||||
Reference in New Issue
Block a user