Retain processing evidence only for selected outputs

This commit is contained in:
2026-11-16 15:08:02 +00:00
parent 5167154a1c
commit 891d4aac2c
21 changed files with 494 additions and 229 deletions

View File

@@ -79,3 +79,9 @@ own phase and available operation coordinates, and peer interruptions and later
file failures preserve their own attribution. The correction ledger joins one file failures preserve their own attribution. The correction ledger joins one
candidate row by module instance and proposal index, preserving repair lineage candidate row by module instance and proposal index, preserving repair lineage
and the recorded validator execution class. and the recorded validator execution class.
The invocation selects retained evidence once. Summary-only processing
accumulates counters as completed sections leave the work window. Ordinary
reports retain the full work matrix and operation facts. Debug retains
candidate and decision history, repair lineage, and native observations.
Without debug, the runtime skips prepared message snapshots.

View File

@@ -658,6 +658,22 @@ Audit #32 and supporting work for #30, #33, and #44.
Resource measurements support reduction of waste without setting a Resource measurements support reduction of waste without setting a
constant-memory or disk-use requirement. constant-memory or disk-use requirement.
**Complete.** The CLI selects ordinary report and debug evidence before
processing. Section outcomes feed shared totals as they drain. The runner
retains the full work grid only for reports, and retains candidate and
validator history only for debug in the CLI. Duplicate module operations and
unused proposal totals were removed. Prepared messages are captured only for
interaction diagnostics. Mode comparison tests and the full offline suite
passed.
A synthetic 4,000-section run with recoverable failures, proposal recoveries,
and apply-time filtering retained about 0.46 MB with summary only, 2.73 MB
with ordinary reports, 1.56 MB with debug evidence, and 3.83 MB with both
after garbage collection. Allocations per run were approximately 20.9,
22.0, 24.0, and 25.0 MB respectively. These runner measurements exclude
provider calls and disk artifacts. All four modes agreed on the transcript
and counters. Temporary measurement instrumentation was removed.
## Stage 19 — Emit native processing progress and update operator guidance ## Stage 19 — Emit native processing progress and update operator guidance
Audit #5, #57, and the operator-facing parts of #8, #24 and #25. Audit #5, #57, and the operator-facing parts of #8, #24 and #25.

View File

@@ -19,6 +19,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
cfg := &loaded.Configuration.Config cfg := &loaded.Configuration.Config
specs := prepared.Prepared.Specs specs := prepared.Prepared.Specs
debug := prepared.Prepared.Debug debug := prepared.Prepared.Debug
evidence := runner.EvidenceSelection{Reports: cfg.Reporting.Enabled, Debug: debug != nil}
var observer *llm.ObservationCollector var observer *llm.ObservationCollector
var hooks llm.CallHooks var hooks llm.CallHooks
if debug != nil { if debug != nil {
@@ -57,7 +58,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
} }
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{ output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs, Config: cfg, Transcript: normalized, ModuleSpecs: specs,
Interactions: interactions, CaptureCandidates: debug != nil, Interactions: interactions, Evidence: &evidence,
}) })
if debug != nil { if debug != nil {
for _, warning := range writer.Warnings() { for _, warning := range writer.Warnings() {
@@ -68,6 +69,6 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output}))) warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
} }
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !cfg.Reporting.Enabled}) ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !evidence.Reports})
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
} }

View File

@@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var terminal pk.ExecutionEvent var terminal pk.ExecutionEvent
var terminalAvailable bool var terminalAvailable bool
var prepared *structuredoutput.PreparedSnapshot var prepared *structuredoutput.PreparedSnapshot
var prompt structuredoutput.PromptMetadata
var record structuredoutput.CallRecord var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil { if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1) record.CallID = c.runtime.nextCallID.Add(1)
@@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
response.Execution.HTTPStatus = &status response.Execution.HTTPStatus = &status
} }
} }
if prepared != nil { response.Prepared = prepared
response.Prepared = prepared response.Prompt = prompt
response.Prompt = prepared.Prompt response.Model = prompt.Model
response.Model = prepared.Prompt.Model
}
mu.Unlock() mu.Unlock()
if result != nil { if result != nil {
response.FinalContent = result.RawOutput if req.CaptureDiagnostics {
response.FinalContent = result.RawOutput
}
response.Model = result.ModelName response.Model = result.ModelName
} }
err := c.finishResponse(parent, &response, result, runErr, out) err := c.finishResponse(parent, &response, result, runErr, out)
if onCall := c.hooks.OnCall; onCall != nil { if onCall := c.hooks.OnCall; onCall != nil {
if response.Prepared != nil { if response.Prompt.PromptID != "" {
record.Prompt = response.Prompt record.Prompt = response.Prompt
} }
record.Finished = time.Now().UTC() record.Finished = time.Now().UTC()
@@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
c.hooks.OnEventWithCall(record.CallID, record.Context, event) c.hooks.OnEventWithCall(record.CallID, record.Context, event)
} }
}), }),
OnPrepared: func(_ context.Context, details pk.PreparedRun) { }
snapshot := preparedSnapshot(details) if req.CaptureDiagnostics || c.hooks.OnCall != nil {
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
metadata := promptMetadata(details)
var snapshot *structuredoutput.PreparedSnapshot
if req.CaptureDiagnostics {
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
}
mu.Lock() mu.Lock()
prepared = snapshot prepared, prompt = snapshot, metadata
mu.Unlock() mu.Unlock()
}, }
} }
return call, nil return call, nil
} }
@@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
} }
} }
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
messages := convertMessages(details.Messages)
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message { func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages)) result := make([]structuredoutput.Message, len(messages))
for i, message := range messages { for i, message := range messages {

View File

@@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"} vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
} }
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}} return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
} }
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {

View File

@@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turns == 2 { if turns == 2 {
return nil, nil return nil, nil
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
}) })
if err != nil || exhausted || turns != 2 { if err != nil || exhausted || turns != 2 {
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err) t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
@@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
if turn.Response != nil { if turn.Response != nil {
return nil, turn.Err return nil, turn.Err
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
} }
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent) _, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") { if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
@@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T)
result = turn.Response result = turn.Response
return nil, turn.Err return nil, turn.Err
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
}) })
if err != nil || exhausted || result == nil { if err != nil || exhausted || result == nil {
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err) t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)

View File

@@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
return nil, nil return nil, nil
} }
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
}) })
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 { if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err) t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
@@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
} }
return nil, turn.Err return nil, turn.Err
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
}) })
if !errors.Is(err, context.Canceled) || deliveries != 1 { if !errors.Is(err, context.Canceled) || deliveries != 1 {
t.Fatalf("deliveries=%d err=%v", deliveries, err) t.Fatalf("deliveries=%d err=%v", deliveries, err)
@@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{}) _, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
return nil, err return nil, err
} }
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
}) })
summary := Summarize(collector.Snapshot()) summary := Summarize(collector.Snapshot())
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete { if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {

View File

@@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
var result map[string]any var result map[string]any
capturing := request capturing := request
capturing.CaptureExchange = true capturing.CaptureExchange = true
capturing.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), capturing, &result) response, err := client.CompleteStructured(context.Background(), capturing, &result)
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) { if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
t.Fatalf("captured final response: %+v", response.Exchange) t.Fatalf("captured final response: %+v", response.Exchange)
@@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
req.CaptureDiagnostics = true
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{}) response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -121,3 +121,57 @@ func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason {
} }
return reasons return reasons
} }
// WorkTotals retains the facts shared by the CLI summary and ordinary reports.
type WorkTotals struct {
Total int
Started int
Completed int
WithErrors int
Corrections int
ModelOperations int
ProposalResponses int
Recoveries int
FilteringResponses int
}
func (t *WorkTotals) Record(state WorkState, operations []Operation) {
if state != WorkNotStarted {
t.Started++
}
if state == WorkCompleted {
t.Completed++
}
hasError := false
for _, operation := range operations {
if operation.Reference.Kind != OperationValidationInput {
t.ModelOperations++
}
if operation.Succeeded {
if operation.Reference.Kind == OperationProposal {
t.ProposalResponses++
if len(operation.Filtered) > 0 {
t.FilteringResponses++
}
}
if len(operation.RecoveryReasons) > 0 {
t.Recoveries++
}
}
hasError = hasError || operation.FailureClass == FailureRecoverable
}
if hasError {
t.WithErrors++
}
}
func (t *WorkTotals) Add(module WorkTotals) {
t.Started += module.Started
t.Completed += module.Completed
t.WithErrors += module.WithErrors
t.Corrections += module.Corrections
t.ModelOperations += module.ModelOperations
t.ProposalResponses += module.ProposalResponses
t.Recoveries += module.Recoveries
t.FilteringResponses += module.FilteringResponses
}

View File

@@ -74,7 +74,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}} cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{ result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}}, Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Evidence: &runner.EvidenceSelection{Reports: true, Debug: true},
}) })
if !errors.Is(err, origin) || client.calls != 3 { if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls) t.Fatalf("fatal cause/calls: %v %d", err, client.calls)

View File

@@ -66,59 +66,35 @@ func Build(input BuildInput) reporting.Snapshot {
return units[i].Section.Index < units[j].Section.Index return units[i].Section.Index < units[j].Section.Index
}) })
} }
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0 var measured outcomes.WorkTotals
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0 if input.RunOutput.Totals != nil {
measured = *input.RunOutput.Totals
} else {
measured.Total = len(units)
for _, unit := range units {
measured.Record(unit.State, unit.Operations)
measured.Corrections += unit.CorrectionsApplied
}
}
for _, unit := range units { for _, unit := range units {
if unit.State == outcomes.WorkNotStarted { if !details {
notStarted++ break
} else {
started++
}
if unit.State == outcomes.WorkCompleted {
completed++
}
if unit.State == outcomes.WorkInterrupted {
interrupted++
}
corrections += unit.CorrectionsApplied
operations := unit.Operations
if details {
operations = append([]outcomes.Operation(nil), operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
} }
operations := append([]outcomes.Operation(nil), unit.Operations...)
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
var failures []reporting.Failure var failures []reporting.Failure
unitHasError := false unitHasError := false
unitRecoveries := 0 unitRecoveries := 0
for _, operation := range operations { for _, operation := range operations {
if operation.Reference.Kind != outcomes.OperationValidationInput {
llmOperations++
}
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
proposalResponses++
}
if operation.Succeeded && len(operation.RecoveryReasons) > 0 { if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
recoveryCount++
unitRecoveries++ unitRecoveries++
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal { snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
filteringResponses++
}
if details {
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
}
} }
if operation.FailureClass == outcomes.FailureRecoverable { if operation.FailureClass == outcomes.FailureRecoverable {
unitHasError = true unitHasError = true
if details { failures = append(failures, mapFailure(operation))
failures = append(failures, mapFailure(operation))
}
} }
} }
if unitHasError {
withErrors++
}
if !details {
continue
}
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance, row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID, ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries} State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
@@ -153,12 +129,13 @@ func Build(input BuildInput) reporting.Snapshot {
return a.ProposalIndex < b.ProposalIndex return a.ProposalIndex < b.ProposalIndex
}) })
runErrors := len(failures) runErrors := len(failures)
interrupted, notStarted := measured.Started-measured.Completed, measured.Total-measured.Started
snapshot.Summary = reporting.Summary{ snapshot.Summary = reporting.Summary{
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed, WorkUnitsTotal: &measured.Total, WorkUnitsStarted: &measured.Started, WorkUnitsCompleted: &measured.Completed,
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &withErrors, WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: &notStarted, WorkUnitsWithErrors: &measured.WithErrors,
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations, RunErrors: &runErrors, CorrectionsApplied: &measured.Corrections, LLMOperations: &measured.ModelOperations,
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount, ProposalResponses: &measured.ProposalResponses, Recoveries: &measured.Recoveries,
ProposalResponsesWithFiltering: &filteringResponses, ProposalResponsesWithFiltering: &measured.FilteringResponses,
} }
} else if len(failures) > 0 { } else if len(failures) > 0 {
runErrors := len(failures) runErrors := len(failures)

View File

@@ -82,9 +82,10 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
) )
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)} coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{ completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
CaptureExchange: req.CaptureExchange, CaptureExchange: req.CaptureExchange,
PromptID: req.PromptID, CaptureDiagnostics: req.Interactions != nil,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey, PromptID: req.PromptID,
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
Context: coordinates, Context: coordinates,
}, &response) }, &response)
sectionIndex := 0 sectionIndex := 0

View File

@@ -207,7 +207,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0 cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, CaptureCandidates: true, Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
}) })

View File

@@ -0,0 +1,94 @@
package runner
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
func evidenceFixture(size int) (*Runner, RunInput) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
for i := range transcript.Segments {
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
}
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
}
return validators.Result{Decisions: decisions}, nil
}}
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
category := req.Section.Index % 4
if category == 0 {
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
}
operation.Succeeded = true
original := "word"
if category == 1 {
original = "absent"
}
if category == 2 {
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
}
return contracts.ProposalResult{
Operations: []outcomes.Operation{operation},
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
}, nil
},
}
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
}
}
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
r, input := evidenceFixture(40)
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
var baseline *schema.Transcript
for _, tc := range []struct {
name string
selection EvidenceSelection
units int
candidates int
}{
{name: "summary"},
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
} {
t.Run(tc.name, func(t *testing.T) {
input.Evidence = &tc.selection
out, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if out.Totals == nil || *out.Totals != want {
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
}
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
}
if baseline == nil {
baseline = out.FinalTranscript
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
t.Fatal("output selection changed final corrections")
}
})
}
}

View File

@@ -161,7 +161,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
} }
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1 proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, CaptureDiagnostics: debug, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
} }
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) { exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response == nil { if turn.Response == nil {

View File

@@ -45,8 +45,6 @@ type ModuleResult struct {
ModuleInstance string `json:"module_instance"` ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"` ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"` Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"` ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"` ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"` AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
@@ -95,13 +93,32 @@ type ValidatorRejectedChange struct {
Message string `json:"message"` Message string `json:"message"`
} }
// EvidenceSelection selects the detailed views consumed by the invocation.
type EvidenceSelection struct {
Reports bool
Debug bool
}
type evidenceNeeds struct {
reports bool
decisions bool
candidates bool
}
func selectEvidence(selection *EvidenceSelection) evidenceNeeds {
if selection == nil {
return evidenceNeeds{reports: true, decisions: true}
}
return evidenceNeeds{reports: selection.Reports, decisions: selection.Debug, candidates: selection.Debug}
}
// RunInput is the deterministic runner input. // RunInput is the deterministic runner input.
type RunInput struct { type RunInput struct {
Config *config.Config Config *config.Config
Transcript *schema.Transcript Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink Interactions structuredoutput.InteractionSink
CaptureCandidates bool Evidence *EvidenceSelection
} }
// RunOutput is the deterministic runner output. // RunOutput is the deterministic runner output.
@@ -109,6 +126,7 @@ type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"` FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"` ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"` Sections []contracts.SectionMetadata `json:"sections"`
Totals *outcomes.WorkTotals `json:"-"`
InventoryAvailable bool `json:"-"` InventoryAvailable bool `json:"-"`
WorkUnits []WorkUnitOutcome `json:"work_units"` WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"` Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
@@ -132,54 +150,70 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil { if err != nil {
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)} return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
} }
inventory := make([]contracts.SectionMetadata, len(sections)) evidence := selectEvidence(input.Evidence)
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections)) totals := outcomes.WorkTotals{Total: len(input.ModuleSpecs) * len(sections)}
var inventory []contracts.SectionMetadata
var workUnits []WorkUnitOutcome
var candidates []proposals.EnrichedCorrectionProposal var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections { if evidence.reports {
inventory[i] = contracts.SectionMetadataFromSection(section) inventory = make([]contracts.SectionMetadata, len(sections))
} for i, section := range sections {
for moduleIndex, spec := range input.ModuleSpecs { inventory[i] = contracts.SectionMetadataFromSection(section)
for _, section := range inventory { }
workUnits = append(workUnits, WorkUnitOutcome{ workUnits = make([]WorkUnitOutcome, 0, totals.Total)
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, for moduleIndex, spec := range input.ModuleSpecs {
ModuleInstance: spec.InstanceName, Section: section, for _, section := range inventory {
State: outcomes.WorkNotStarted, workUnits = append(workUnits, newWorkUnit(spec, section, moduleIndex))
}) }
} }
} }
snapshot := func() RunOutput { snapshot := func() RunOutput {
return RunOutput{ return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results, FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, InventoryAvailable: true, WorkUnits: workUnits, Candidates: candidates, Sections: inventory, InventoryAvailable: true, Totals: &totals, WorkUnits: workUnits, Candidates: candidates,
} }
} }
for moduleIndex, spec := range input.ModuleSpecs { for moduleIndex, spec := range input.ModuleSpecs {
execution, err := r.executeModule(ctx, spec, sections, input.Interactions) var reportUnits []WorkUnitOutcome
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)] if evidence.reports {
copy(units, execution.pipeline.WorkUnits) reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
if err == nil && ctx.Err() != nil {
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
} }
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
units := execution.pipeline.WorkUnits
if err == nil { if err == nil {
previous := sections filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
sections = execution.updated if canceled := ctx.Err(); canceled != nil {
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units) err = locateFailure(canceled, "application", failureReference(spec, nil, ""), false, nil)
} else {
execution.pipeline.recordApplyFiltering(filters, evidence.reports)
sections = execution.updated
execution.pipeline.Totals.Started = len(sections)
execution.pipeline.Totals.Completed = len(sections)
execution.pipeline.Totals.Corrections = len(execution.applied)
}
} }
var failure *ExecutionFailure var failure *ExecutionFailure
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application" applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
appliedIndices := make(map[int]struct{}, len(execution.applied)) if applicationInterrupted {
for _, change := range execution.applied { execution.pipeline.Totals.Completed = 0
appliedIndices[change.ProposalIndex] = struct{}{} }
totals.Add(execution.pipeline.Totals)
appliedIndices := make(map[int]struct{})
if evidence.reports || evidence.candidates {
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
} }
var operations []outcomes.Operation
for i := range units { for i := range units {
unit := &units[i] unit := &units[i]
unit.ModuleIndex = moduleIndex unit.ModuleIndex = moduleIndex
if err == nil { if err == nil {
unit.State = outcomes.WorkCompleted unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = execution.appliedBySection[i] if evidence.reports {
unit.CorrectionsApplied = execution.appliedBySection[unit.Section.Index]
}
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted { } else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
unit.State = outcomes.WorkInterrupted unit.State = outcomes.WorkInterrupted
} }
@@ -196,16 +230,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
branch.Outcome = "approved_skipped" branch.Outcome = "approved_skipped"
} }
} }
operations = append(operations, unit.Operations...)
} }
if !evidence.reports && evidence.candidates {
if input.CaptureCandidates { workUnits = append(workUnits, units...)
candidates = append(candidates, execution.pipeline.Candidates...)
} }
candidates = append(candidates, execution.pipeline.Candidates...)
result := ModuleResult{ result := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy, ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected, ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
} }
if err != nil { if err != nil {
@@ -213,8 +245,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
result.ErrorMessage = err.Error() result.ErrorMessage = err.Error()
} else { } else {
result.Status = ModuleStatusSuccess result.Status = ModuleStatusSuccess
result.AppliedChanges = execution.applied if evidence.decisions {
result.SkippedChanges = execution.skipped result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
}
} }
results = append(results, result) results = append(results, result)
if err != nil { if err != nil {
@@ -237,7 +271,7 @@ type moduleExecution struct {
appliedBySection []int appliedBySection []int
} }
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) { func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
var execution moduleExecution var execution moduleExecution
origin := failureReference(spec, nil, "") origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
@@ -253,6 +287,8 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
} }
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{ execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
Module: module, Module: module,
Evidence: &evidence,
ReportUnits: reportUnits,
Spec: spec, Spec: spec,
Policy: execution.policy, Policy: execution.policy,
Sections: sections, Sections: sections,
@@ -323,6 +359,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
} }
type collectSectionProposalsInput struct { type collectSectionProposalsInput struct {
Evidence *evidenceNeeds
ReportUnits []WorkUnitOutcome
Interactions structuredoutput.InteractionSink Interactions structuredoutput.InteractionSink
Module contracts.TranscriptModule Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec Spec contracts.ModuleRunSpec
@@ -356,16 +394,18 @@ type sectionValidationResult struct {
} }
type sectionWork struct { type sectionWork struct {
unit WorkUnitOutcome
candidates []proposals.EnrichedCorrectionProposal candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult validation *sectionValidationResult
finished bool finished bool
} }
type modulePipelineResult struct { type modulePipelineResult struct {
ProposalCount int Totals outcomes.WorkTotals
proposalFacts map[int]proposalRecovery
revised map[int]struct{}
Candidates []proposals.EnrichedCorrectionProposal Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome WorkUnits []WorkUnitOutcome
ValidatorDecisions []ValidatorDecisionRecord ValidatorDecisions []ValidatorDecisionRecord
ValidatorRejected []ValidatorRejectedChange ValidatorRejected []ValidatorRejectedChange
@@ -410,17 +450,16 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
} }
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) { func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
out := modulePipelineResult{ var out modulePipelineResult
Approved: make([]proposals.EnrichedCorrectionProposal, 0), evidence := selectEvidence(nil)
Operations: make([]outcomes.Operation, 0), if input.Evidence != nil {
WorkUnits: make([]WorkUnitOutcome, len(input.Sections)), evidence = *input.Evidence
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
ValidatorRejected: make([]ValidatorRejectedChange, 0),
} }
for i, section := range input.Sections { out.WorkUnits = input.ReportUnits
out.WorkUnits[i] = WorkUnitOutcome{ if evidence.reports && out.WorkUnits == nil {
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID, out.WorkUnits = make([]WorkUnitOutcome, len(input.Sections))
Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted, for i, section := range input.Sections {
out.WorkUnits[i] = newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(section), 0)
} }
} }
@@ -487,17 +526,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return sectionEnriched return sectionEnriched
} }
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) { recordCandidates := func(unit *WorkUnitOutcome, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
unit.ProposalIndexOffset = nextProposalIndex unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched { if evidence.candidates {
candidate.ProposalIndex += nextProposalIndex for _, candidate := range enriched {
out.Candidates = append(out.Candidates, candidate) candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
} }
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations { for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex] operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded { if evidence.reports && operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched { for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex) operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
} }
@@ -516,16 +555,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
interactions = capture interactions = capture
} }
validationInput := validateSectionCandidatesInput{ validationInput := validateSectionCandidatesInput{
Spec: input.Spec, Spec: input.Spec,
Policy: input.Policy, SuppressDecisionHistory: !evidence.decisions,
WorkingTranscript: transcriptFromSection(sectionResult.section), Policy: input.Policy,
SectionIndex: &sectionMetadata.Index, WorkingTranscript: transcriptFromSection(sectionResult.section),
Validators: validatorsOrdered, SectionIndex: &sectionMetadata.Index,
SectionEnriched: enriched, Validators: validatorsOrdered,
Exchange: sectionResult.proposal.Exchange, SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient, Exchange: sectionResult.proposal.Exchange,
GlossaryView: input.GlossaryView, ValidationLLMClient: input.ValidationClient,
Interactions: interactions, GlossaryView: input.GlossaryView,
Interactions: interactions,
} }
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 { if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript) validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
@@ -555,8 +595,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
failure: failure, failure: failure,
} }
} }
recordValidation := func(res sectionValidationResult) { recordValidation := func(unit *WorkUnitOutcome, res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
unit.Operations = append(unit.Operations, res.operations...) unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered) mergeFilteredProposals(unit, res.filtered)
@@ -567,18 +606,30 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.State = outcomes.WorkInterrupted unit.State = outcomes.WorkInterrupted
} }
out.Approved = append(out.Approved, res.approved...) out.Approved = append(out.Approved, res.approved...)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...) if evidence.decisions {
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...) out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
if len(res.approved) > 0 {
out.rememberProposalRecovery(unit)
}
} }
nextAdmission, active := 0, 0 nextAdmission, active := 0, 0
for { for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] { for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates) recordCandidates(&work.unit, work.candidates)
if work.validation != nil { if work.validation != nil {
work.validation.assignProposalIndices(offset) work.validation.assignProposalIndices(offset)
recordValidation(*work.validation) recordValidation(&work.unit, *work.validation)
}
out.Totals.Record(work.unit.State, work.unit.Operations)
if evidence.reports {
out.WorkUnits[nextSectionToRecord] = work.unit
} else if evidence.candidates && len(work.unit.Repairs) > 0 {
work.unit.Operations = nil
out.WorkUnits = append(out.WorkUnits, work.unit)
} }
delete(pending, nextSectionToRecord) delete(pending, nextSectionToRecord)
nextSectionToRecord++ nextSectionToRecord++
@@ -586,7 +637,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil { for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission sectionPos := nextAdmission
nextAdmission++ nextAdmission++
pending[sectionPos] = &sectionWork{} pending[sectionPos] = &sectionWork{unit: newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(input.Sections[sectionPos]), 0)}
active++ active++
workers.Go(func() { workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos) proposed := proposeSection(runCtx, input, sectionPos)
@@ -600,13 +651,13 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
select { select {
case result := <-proposalResults: case result := <-proposalResults:
active-- active--
unit := &out.WorkUnits[result.sectionPos] work := pending[result.sectionPos]
unit := &work.unit
unit.Operations = append(unit.Operations, result.proposal.Operations...) unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started { if result.started {
unit.State = outcomes.WorkInterrupted unit.State = outcomes.WorkInterrupted
} }
unit.Failure = result.failure unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil { if result.started && result.failure == nil {
work.candidates = makeCandidates(result) work.candidates = makeCandidates(result)
if runCtx.Err() == nil { if runCtx.Err() == nil {
@@ -633,10 +684,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
}) })
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool { sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex { if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
@@ -668,19 +715,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
} }
type validateSectionCandidatesInput struct { type validateSectionCandidatesInput struct {
Section *validators.SectionLookup SuppressDecisionHistory bool
RepairProposalIndex *int Section *validators.SectionLookup
RepairAttempt *int RepairProposalIndex *int
Exchange *structuredoutput.Exchange RepairAttempt *int
Spec contracts.ModuleRunSpec Exchange *structuredoutput.Exchange
Policy proposals.ReplacementPolicy Spec contracts.ModuleRunSpec
WorkingTranscript *schema.Transcript Policy proposals.ReplacementPolicy
SectionIndex *int WorkingTranscript *schema.Transcript
Validators []validators.Validator SectionIndex *int
SectionEnriched []proposals.EnrichedCorrectionProposal Validators []validators.Validator
ValidationLLMClient contracts.StructuredLLMClient SectionEnriched []proposals.EnrichedCorrectionProposal
Interactions structuredoutput.InteractionSink ValidationLLMClient contracts.StructuredLLMClient
GlossaryView *validators.GlossaryView Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
} }
type validateSectionCandidatesResult struct { type validateSectionCandidatesResult struct {
@@ -781,16 +829,18 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
noEffect[d.ProposalIndex] = true noEffect[d.ProposalIndex] = true
} }
} }
decisions = append(decisions, ValidatorDecisionRecord{ if !input.SuppressDecisionHistory {
ValidatorName: validator.Name(), decisions = append(decisions, ValidatorDecisionRecord{
ExecutionClass: class, ValidatorName: validator.Name(),
ProposalIndex: d.ProposalIndex, ExecutionClass: class,
Approved: approved, ProposalIndex: d.ProposalIndex,
Disposition: disposition, Approved: approved,
ReasonCode: d.ReasonCode, Disposition: disposition,
Message: d.Message, ReasonCode: d.ReasonCode,
DiagnosticArtifactPath: d.DiagnosticArtifactPath, Message: d.Message,
}) DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
}
if approved { if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex]) nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue continue
@@ -800,19 +850,21 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1}) filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
} }
} }
p := byIndex[d.ProposalIndex] if feedback || !input.SuppressDecisionHistory {
rejected = append(rejected, ValidatorRejectedChange{ p := byIndex[d.ProposalIndex]
ValidatorName: validator.Name(), rejected = append(rejected, ValidatorRejectedChange{
ProposalIndex: p.ProposalIndex, ValidatorName: validator.Name(),
ModuleKey: p.ModuleKey, ProposalIndex: p.ProposalIndex,
ModuleInstance: p.ModuleInstance, ModuleKey: p.ModuleKey,
TargetSegmentID: p.TargetSegmentID, ModuleInstance: p.ModuleInstance,
OriginalText: p.OriginalText, TargetSegmentID: p.TargetSegmentID,
CorrectedText: p.CorrectedText, OriginalText: p.OriginalText,
Disposition: disposition, CorrectedText: p.CorrectedText,
ReasonCode: d.ReasonCode, Disposition: disposition,
Message: d.Message, ReasonCode: d.ReasonCode,
}) Message: d.Message,
})
}
} }
if err != nil { if err != nil {
@@ -877,9 +929,9 @@ func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes.
} }
} }
func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) { func findApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy) map[int][]outcomes.FilteredProposal {
if len(skipped) == 0 { if len(skipped) == 0 {
return return nil
} }
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved)) byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
sectionPosition := make(map[int]int, len(sections)) sectionPosition := make(map[int]int, len(sections))
@@ -916,26 +968,7 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
} }
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1}) filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
} }
for pos, items := range filtered { return filtered
unit := &units[pos]
revisions := make(map[int]*outcomes.Operation)
for i := range unit.Operations {
op := &unit.Operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil &&
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
revisions[*op.Reference.ProposalIndex] = op
}
}
original := items[:0]
for _, item := range items {
if op := revisions[item.ProposalIndex]; op != nil {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
} else {
original = append(original, item)
}
}
mergeFilteredProposals(unit, original)
}
} }
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) { func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {

View File

@@ -173,7 +173,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
if len(without.Candidates) != 0 { if len(without.Candidates) != 0 {
t.Fatalf("debug-off candidate capture: %+v", without.Candidates) t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
} }
input.CaptureCandidates = true input.Evidence = &EvidenceSelection{Reports: true, Debug: true}
with, err := r.Run(context.Background(), input) with, err := r.Run(context.Background(), input)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -844,7 +844,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
}, },
}, },
}}) }})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true}) out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -973,8 +973,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 { if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults) t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
} }
if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid { if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations) t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations)
} }
} }

View File

@@ -218,7 +218,7 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
cfg.Chunking.MaxSectionTokens = 1 cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0 cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{ out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true, Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation, ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,

View File

@@ -0,0 +1,78 @@
package runner
import (
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
)
func newWorkUnit(spec contracts.ModuleRunSpec, section contracts.SectionMetadata, moduleIndex int) WorkUnitOutcome {
return WorkUnitOutcome{
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section, State: outcomes.WorkNotStarted,
}
}
type proposalRecovery struct {
recovered bool
filtered bool
}
// Application can discover additional defects. Keep only the proposal operation
// flags needed to update totals and the identities of approved revisions.
func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) {
for _, operation := range unit.Operations {
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
if p.proposalFacts == nil {
p.proposalFacts = make(map[int]proposalRecovery)
}
p.proposalFacts[unit.Section.Index] = proposalRecovery{
recovered: len(operation.RecoveryReasons) > 0, filtered: len(operation.Filtered) > 0,
}
break
}
}
for _, repair := range unit.Repairs {
if repair.Outcome == "approved" {
if p.revised == nil {
p.revised = make(map[int]struct{})
}
p.revised[repair.ProposalIndex] = struct{}{}
}
}
}
func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) {
for pos, items := range filters {
original := items[:0]
for _, item := range items {
if _, repaired := p.revised[item.ProposalIndex]; !repaired {
original = append(original, item)
continue
}
if reports {
operations := p.WorkUnits[pos].Operations
for i := range operations {
op := &operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
*op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
break
}
}
}
}
if facts, ok := p.proposalFacts[pos]; ok && len(original) != 0 {
if !facts.recovered {
p.Totals.Recoveries++
}
if !facts.filtered {
p.Totals.FilteringResponses++
}
if reports {
mergeFilteredProposals(&p.WorkUnits[pos], original)
}
}
}
}

View File

@@ -26,11 +26,12 @@ type DiagnosticContext struct {
// Request names an embedded prompt and response schema. Profile selection and // Request names an embedded prompt and response schema. Profile selection and
// credentials are supplied by the role-bound runtime client. // credentials are supplied by the role-bound runtime client.
type Request struct { type Request struct {
PromptID string `json:"prompt_id"` PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"` Variables map[string]string `json:"variables"`
Schema responseschema.Key `json:"schema"` Schema responseschema.Key `json:"schema"`
Context DiagnosticContext `json:"context"` Context DiagnosticContext `json:"context"`
CaptureExchange bool `json:"-"` CaptureExchange bool `json:"-"`
CaptureDiagnostics bool `json:"-"`
} }
// PromptMetadata is the execution snapshot returned by the runtime when available. // PromptMetadata is the execution snapshot returned by the runtime when available.

View File

@@ -130,7 +130,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID, PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: coordinates, Context: coordinates, CaptureDiagnostics: req.Interactions != nil,
}, &response) }, &response)
batchIndex := batch.BatchIndex batchIndex := batch.BatchIndex
operation := outcomes.Operation{ operation := outcomes.Operation{