Adopt PromptKit v0.23.0 conversations and execution accounting
This commit is contained in:
@@ -20,15 +20,16 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type executionEvent struct {
|
||||
@@ -72,7 +73,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
@@ -87,7 +88,8 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
||||
}
|
||||
return struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{calls, events}
|
||||
Summary llm.ExecutionSummary `json:"summary"`
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{llm.Summarize(snapshot), calls, events}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user