Finish PromptKit repair and execution accounting follow-ups
This commit is contained in:
@@ -68,12 +68,13 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
}
|
||||
for _, call := range snapshot.Calls {
|
||||
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]}
|
||||
summary := llm.Summarize(group)
|
||||
var finished *time.Time
|
||||
if !call.Finished.IsZero() {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: summary.TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(summary)})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
|
||||
@@ -15,13 +15,14 @@ type RepairResponse struct {
|
||||
}
|
||||
|
||||
// DecodeRepairResponse checks correspondence after structural schema validation.
|
||||
// It returns the decoded response for diagnostics when correspondence fails.
|
||||
func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) {
|
||||
var response RepairResponse
|
||||
if err := json.Unmarshal(content, &response); err != nil {
|
||||
return RepairResponse{}, invalidRepairResponse(err)
|
||||
}
|
||||
if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) {
|
||||
return RepairResponse{}, invalidRepairResponse(nil)
|
||||
return response, invalidRepairResponse(nil)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
@@ -20,11 +20,14 @@ func TestDecodeRepairResponse(t *testing.T) {
|
||||
{"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := DecodeRepairResponse([]byte(tc.content), 2)
|
||||
response, err := DecodeRepairResponse([]byte(tc.content), 2)
|
||||
if tc.valid && err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !tc.valid {
|
||||
if response.ProposalRef == 0 || response.Action == "" {
|
||||
t.Fatalf("missing decoded evidence: %+v", response)
|
||||
}
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable {
|
||||
t.Fatalf("failure = %v", err)
|
||||
|
||||
@@ -2,7 +2,6 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
@@ -151,9 +150,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
completion, callErr := *turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
if callErr == nil {
|
||||
_ = json.Unmarshal(completion.Content, &response)
|
||||
}
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
@@ -202,7 +198,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
return nil, callErr
|
||||
}
|
||||
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
var err error
|
||||
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
operation.FailureReason = outcomes.ReasonInvalidOutput
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
@@ -211,12 +208,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
if decoded.Action == "withdraw" {
|
||||
if response.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
return nil, nil
|
||||
}
|
||||
revision := decoded.Correction
|
||||
revision := response.Correction
|
||||
localRef = revision.SegmentReference
|
||||
target := 0
|
||||
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {
|
||||
|
||||
Reference in New Issue
Block a user