Retain processing evidence only for selected outputs
This commit is contained in:
@@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
var terminal pk.ExecutionEvent
|
||||
var terminalAvailable bool
|
||||
var prepared *structuredoutput.PreparedSnapshot
|
||||
var prompt structuredoutput.PromptMetadata
|
||||
var record structuredoutput.CallRecord
|
||||
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
||||
record.CallID = c.runtime.nextCallID.Add(1)
|
||||
@@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
response.Execution.HTTPStatus = &status
|
||||
}
|
||||
}
|
||||
if prepared != nil {
|
||||
response.Prepared = prepared
|
||||
response.Prompt = prepared.Prompt
|
||||
response.Model = prepared.Prompt.Model
|
||||
}
|
||||
response.Prepared = prepared
|
||||
response.Prompt = prompt
|
||||
response.Model = prompt.Model
|
||||
mu.Unlock()
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
if req.CaptureDiagnostics {
|
||||
response.FinalContent = result.RawOutput
|
||||
}
|
||||
response.Model = result.ModelName
|
||||
}
|
||||
err := c.finishResponse(parent, &response, result, runErr, out)
|
||||
if onCall := c.hooks.OnCall; onCall != nil {
|
||||
if response.Prepared != nil {
|
||||
if response.Prompt.PromptID != "" {
|
||||
record.Prompt = response.Prompt
|
||||
}
|
||||
record.Finished = time.Now().UTC()
|
||||
@@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
||||
}
|
||||
}),
|
||||
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
|
||||
snapshot := preparedSnapshot(details)
|
||||
}
|
||||
if req.CaptureDiagnostics || c.hooks.OnCall != nil {
|
||||
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
|
||||
metadata := promptMetadata(details)
|
||||
var snapshot *structuredoutput.PreparedSnapshot
|
||||
if req.CaptureDiagnostics {
|
||||
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
|
||||
}
|
||||
mu.Lock()
|
||||
prepared = snapshot
|
||||
prepared, prompt = snapshot, metadata
|
||||
mu.Unlock()
|
||||
},
|
||||
}
|
||||
}
|
||||
return call, nil
|
||||
}
|
||||
@@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
|
||||
messages := convertMessages(details.Messages)
|
||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
||||
}
|
||||
|
||||
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
|
||||
result := make([]structuredoutput.Message, len(messages))
|
||||
for i, message := range messages {
|
||||
|
||||
@@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||
id = prompts.PromptIDValidatorEditorialReview
|
||||
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
|
||||
}
|
||||
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||
return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
|
||||
if turns == 2 {
|
||||
return nil, nil
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
|
||||
})
|
||||
if err != nil || exhausted || turns != 2 {
|
||||
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
|
||||
@@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
|
||||
if turn.Response != nil {
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
|
||||
}
|
||||
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
|
||||
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
|
||||
@@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T)
|
||||
result = turn.Response
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
|
||||
})
|
||||
if err != nil || exhausted || result == nil {
|
||||
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)
|
||||
|
||||
@@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
|
||||
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
|
||||
@@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
|
||||
}
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || deliveries != 1 {
|
||||
t.Fatalf("deliveries=%d err=%v", deliveries, err)
|
||||
@@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
|
||||
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
|
||||
return nil, err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
summary := Summarize(collector.Snapshot())
|
||||
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
|
||||
|
||||
@@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
|
||||
var result map[string]any
|
||||
capturing := request
|
||||
capturing.CaptureExchange = true
|
||||
capturing.CaptureDiagnostics = true
|
||||
response, err := client.CompleteStructured(context.Background(), capturing, &result)
|
||||
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
|
||||
t.Fatalf("captured final response: %+v", response.Exchange)
|
||||
@@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.CaptureDiagnostics = true
|
||||
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user