Retain processing evidence only for selected outputs

This commit is contained in:
2026-11-16 15:08:02 +00:00
parent 5167154a1c
commit 891d4aac2c
21 changed files with 494 additions and 229 deletions

View File

@@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var terminal pk.ExecutionEvent
var terminalAvailable bool
var prepared *structuredoutput.PreparedSnapshot
var prompt structuredoutput.PromptMetadata
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
@@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
response.Execution.HTTPStatus = &status
}
}
if prepared != nil {
response.Prepared = prepared
response.Prompt = prepared.Prompt
response.Model = prepared.Prompt.Model
}
response.Prepared = prepared
response.Prompt = prompt
response.Model = prompt.Model
mu.Unlock()
if result != nil {
response.FinalContent = result.RawOutput
if req.CaptureDiagnostics {
response.FinalContent = result.RawOutput
}
response.Model = result.ModelName
}
err := c.finishResponse(parent, &response, result, runErr, out)
if onCall := c.hooks.OnCall; onCall != nil {
if response.Prepared != nil {
if response.Prompt.PromptID != "" {
record.Prompt = response.Prompt
}
record.Finished = time.Now().UTC()
@@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
}
}),
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
snapshot := preparedSnapshot(details)
}
if req.CaptureDiagnostics || c.hooks.OnCall != nil {
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
metadata := promptMetadata(details)
var snapshot *structuredoutput.PreparedSnapshot
if req.CaptureDiagnostics {
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
}
mu.Lock()
prepared = snapshot
prepared, prompt = snapshot, metadata
mu.Unlock()
},
}
}
return call, nil
}
@@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
}
}
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
messages := convertMessages(details.Messages)
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages))
for i, message := range messages {

View File

@@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {

View File

@@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turns == 2 {
return nil, nil
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
})
if err != nil || exhausted || turns != 2 {
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
@@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turn.Response != nil {
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
}
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
@@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T)
result = turn.Response
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
})
if err != nil || exhausted || result == nil {
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)

View File

@@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
return nil, nil
}
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
@@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
}
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if !errors.Is(err, context.Canceled) || deliveries != 1 {
t.Fatalf("deliveries=%d err=%v", deliveries, err)
@@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
return nil, err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
summary := Summarize(collector.Snapshot())
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {

View File

@@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
var result map[string]any
capturing := request
capturing.CaptureExchange = true
capturing.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), capturing, &result)
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
t.Fatalf("captured final response: %+v", response.Exchange)
@@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
req.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)