Finish PromptKit repair and execution accounting follow-ups

This commit is contained in:
2026-11-15 01:44:01 +00:00
parent d6413b3996
commit dfdc6cc2c4
6 changed files with 33 additions and 10 deletions

View File

@@ -243,6 +243,25 @@ func TestSubprocessCorrectionRepairConversation(t *testing.T) {
if len(ledger) != 3 { if len(ledger) != 3 {
t.Fatalf("ledger: %+v", ledger) t.Fatalf("ledger: %+v", ledger)
} }
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
if err != nil || len(paths) != 1 {
t.Fatalf("execution path: %v %v", paths, err)
}
data, err := os.ReadFile(paths[0])
if err != nil {
t.Fatal(err)
}
var execution struct {
Summary struct {
ProviderAttempts int `json:"provider_attempts"`
} `json:"summary"`
}
if err := json.Unmarshal(data, &execution); err != nil {
t.Fatal(err)
}
if got := execution.Summary.ProviderAttempts; got != len(requests) {
t.Fatalf("execution provider attempts = %d, captured requests = %d", got, len(requests))
}
} }
}) })
} }

View File

@@ -7,6 +7,8 @@ and [developer guide](../development.md) for the applicable policies.
## Stage 1 — Finish the PromptKit Adoption Follow-ups ## Stage 1 — Finish the PromptKit Adoption Follow-ups
**Complete.** Focused decoder, runner, reporting, and subprocess checks passed.
1. In the debug case of `TestSubprocessCorrectionRepairConversation`, read the 1. In the debug case of `TestSubprocessCorrectionRepairConversation`, read the
run's `llm-execution.json` and compare its top-level run's `llm-execution.json` and compare its top-level
`summary.provider_attempts` with the number of requests captured by the `summary.provider_attempts` with the number of requests captured by the

View File

@@ -68,12 +68,13 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
} }
for _, call := range snapshot.Calls { for _, call := range snapshot.Calls {
group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]} group := llm.ObservationSnapshot{Calls: []llm.LogicalCall{call}, Events: eventsByCall[call.CallID]}
summary := llm.Summarize(group)
var finished *time.Time var finished *time.Time
if !call.Finished.IsZero() { if !call.Finished.IsZero() {
value := call.Finished value := call.Finished
finished = &value finished = &value
} }
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))}) calls = append(calls, executionCall{TerminalCoverageComplete: summary.TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(summary)})
} }
for _, event := range snapshot.Events { for _, event := range snapshot.Events {
var generation, attempt *uint64 var generation, attempt *uint64

View File

@@ -15,13 +15,14 @@ type RepairResponse struct {
} }
// DecodeRepairResponse checks correspondence after structural schema validation. // DecodeRepairResponse checks correspondence after structural schema validation.
// It returns the decoded response for diagnostics when correspondence fails.
func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) { func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) {
var response RepairResponse var response RepairResponse
if err := json.Unmarshal(content, &response); err != nil { if err := json.Unmarshal(content, &response); err != nil {
return RepairResponse{}, invalidRepairResponse(err) return RepairResponse{}, invalidRepairResponse(err)
} }
if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) { if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) {
return RepairResponse{}, invalidRepairResponse(nil) return response, invalidRepairResponse(nil)
} }
return response, nil return response, nil
} }

View File

@@ -20,11 +20,14 @@ func TestDecodeRepairResponse(t *testing.T) {
{"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false}, {"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := DecodeRepairResponse([]byte(tc.content), 2) response, err := DecodeRepairResponse([]byte(tc.content), 2)
if tc.valid && err != nil { if tc.valid && err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !tc.valid { if !tc.valid {
if response.ProposalRef == 0 || response.Action == "" {
t.Fatalf("missing decoded evidence: %+v", response)
}
var failure *structuredoutput.Failure var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable { if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable {
t.Fatalf("failure = %v", err) t.Fatalf("failure = %v", err)

View File

@@ -2,7 +2,6 @@ package runner
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"path/filepath" "path/filepath"
@@ -151,9 +150,6 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
completion, callErr := *turn.Response, turn.Err completion, callErr := *turn.Response, turn.Err
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1 proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
var response proposal_generation.RepairResponse var response proposal_generation.RepairResponse
if callErr == nil {
_ = json.Unmarshal(completion.Content, &response)
}
operation := outcomes.Operation{ operation := outcomes.Operation{
Reference: outcomes.Reference{ Reference: outcomes.Reference{
@@ -202,7 +198,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} }
return nil, callErr return nil, callErr
} }
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1) var err error
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil { if err != nil {
operation.FailureReason = outcomes.ReasonInvalidOutput operation.FailureReason = outcomes.ReasonInvalidOutput
finishAttempt(err, nil, validateSectionCandidatesResult{}) finishAttempt(err, nil, validateSectionCandidatesResult{})
@@ -211,12 +208,12 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} }
operation.Succeeded = true operation.Succeeded = true
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts) operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
if decoded.Action == "withdraw" { if response.Action == "withdraw" {
finishAttempt(nil, nil, validateSectionCandidatesResult{}) finishAttempt(nil, nil, validateSectionCandidatesResult{})
branch.Outcome = "withdrawn" branch.Outcome = "withdrawn"
return nil, nil return nil, nil
} }
revision := decoded.Correction revision := response.Correction
localRef = revision.SegmentReference localRef = revision.SegmentReference
target := 0 target := 0
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) { if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {