Retain processing evidence only for selected outputs
This commit is contained in:
@@ -79,3 +79,9 @@ own phase and available operation coordinates, and peer interruptions and later
|
|||||||
file failures preserve their own attribution. The correction ledger joins one
|
file failures preserve their own attribution. The correction ledger joins one
|
||||||
candidate row by module instance and proposal index, preserving repair lineage
|
candidate row by module instance and proposal index, preserving repair lineage
|
||||||
and the recorded validator execution class.
|
and the recorded validator execution class.
|
||||||
|
|
||||||
|
The invocation selects retained evidence once. Summary-only processing
|
||||||
|
accumulates counters as completed sections leave the work window. Ordinary
|
||||||
|
reports retain the full work matrix and operation facts. Debug retains
|
||||||
|
candidate and decision history, repair lineage, and native observations.
|
||||||
|
Without debug, the runtime skips prepared message snapshots.
|
||||||
|
|||||||
@@ -658,6 +658,22 @@ Audit #32 and supporting work for #30, #33, and #44.
|
|||||||
Resource measurements support reduction of waste without setting a
|
Resource measurements support reduction of waste without setting a
|
||||||
constant-memory or disk-use requirement.
|
constant-memory or disk-use requirement.
|
||||||
|
|
||||||
|
**Complete.** The CLI selects ordinary report and debug evidence before
|
||||||
|
processing. Section outcomes feed shared totals as they drain. The runner
|
||||||
|
retains the full work grid only for reports, and retains candidate and
|
||||||
|
validator history only for debug in the CLI. Duplicate module operations and
|
||||||
|
unused proposal totals were removed. Prepared messages are captured only for
|
||||||
|
interaction diagnostics. Mode comparison tests and the full offline suite
|
||||||
|
passed.
|
||||||
|
|
||||||
|
A synthetic 4,000-section run with recoverable failures, proposal recoveries,
|
||||||
|
and apply-time filtering retained about 0.46 MB with summary only, 2.73 MB
|
||||||
|
with ordinary reports, 1.56 MB with debug evidence, and 3.83 MB with both
|
||||||
|
after garbage collection. Allocations per run were approximately 20.9,
|
||||||
|
22.0, 24.0, and 25.0 MB respectively. These runner measurements exclude
|
||||||
|
provider calls and disk artifacts. All four modes agreed on the transcript
|
||||||
|
and counters. Temporary measurement instrumentation was removed.
|
||||||
|
|
||||||
## Stage 19 — Emit native processing progress and update operator guidance
|
## Stage 19 — Emit native processing progress and update operator guidance
|
||||||
|
|
||||||
Audit #5, #57, and the operator-facing parts of #8, #24 and #25.
|
Audit #5, #57, and the operator-facing parts of #8, #24 and #25.
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
|||||||
cfg := &loaded.Configuration.Config
|
cfg := &loaded.Configuration.Config
|
||||||
specs := prepared.Prepared.Specs
|
specs := prepared.Prepared.Specs
|
||||||
debug := prepared.Prepared.Debug
|
debug := prepared.Prepared.Debug
|
||||||
|
evidence := runner.EvidenceSelection{Reports: cfg.Reporting.Enabled, Debug: debug != nil}
|
||||||
var observer *llm.ObservationCollector
|
var observer *llm.ObservationCollector
|
||||||
var hooks llm.CallHooks
|
var hooks llm.CallHooks
|
||||||
if debug != nil {
|
if debug != nil {
|
||||||
@@ -57,7 +58,7 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
|||||||
}
|
}
|
||||||
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||||
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||||
Interactions: interactions, CaptureCandidates: debug != nil,
|
Interactions: interactions, Evidence: &evidence,
|
||||||
})
|
})
|
||||||
if debug != nil {
|
if debug != nil {
|
||||||
for _, warning := range writer.Warnings() {
|
for _, warning := range writer.Warnings() {
|
||||||
@@ -68,6 +69,6 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
|||||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||||
}
|
}
|
||||||
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
|
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
|
||||||
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !cfg.Reporting.Enabled})
|
ModuleSpecs: specs, RunOutput: &output, ProcessingError: runErr, SummaryOnly: !evidence.Reports})
|
||||||
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
|||||||
var terminal pk.ExecutionEvent
|
var terminal pk.ExecutionEvent
|
||||||
var terminalAvailable bool
|
var terminalAvailable bool
|
||||||
var prepared *structuredoutput.PreparedSnapshot
|
var prepared *structuredoutput.PreparedSnapshot
|
||||||
|
var prompt structuredoutput.PromptMetadata
|
||||||
var record structuredoutput.CallRecord
|
var record structuredoutput.CallRecord
|
||||||
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
||||||
record.CallID = c.runtime.nextCallID.Add(1)
|
record.CallID = c.runtime.nextCallID.Add(1)
|
||||||
@@ -90,19 +91,19 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
|||||||
response.Execution.HTTPStatus = &status
|
response.Execution.HTTPStatus = &status
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if prepared != nil {
|
response.Prepared = prepared
|
||||||
response.Prepared = prepared
|
response.Prompt = prompt
|
||||||
response.Prompt = prepared.Prompt
|
response.Model = prompt.Model
|
||||||
response.Model = prepared.Prompt.Model
|
|
||||||
}
|
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
if result != nil {
|
if result != nil {
|
||||||
response.FinalContent = result.RawOutput
|
if req.CaptureDiagnostics {
|
||||||
|
response.FinalContent = result.RawOutput
|
||||||
|
}
|
||||||
response.Model = result.ModelName
|
response.Model = result.ModelName
|
||||||
}
|
}
|
||||||
err := c.finishResponse(parent, &response, result, runErr, out)
|
err := c.finishResponse(parent, &response, result, runErr, out)
|
||||||
if onCall := c.hooks.OnCall; onCall != nil {
|
if onCall := c.hooks.OnCall; onCall != nil {
|
||||||
if response.Prepared != nil {
|
if response.Prompt.PromptID != "" {
|
||||||
record.Prompt = response.Prompt
|
record.Prompt = response.Prompt
|
||||||
}
|
}
|
||||||
record.Finished = time.Now().UTC()
|
record.Finished = time.Now().UTC()
|
||||||
@@ -146,12 +147,18 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
|||||||
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
|
}
|
||||||
snapshot := preparedSnapshot(details)
|
if req.CaptureDiagnostics || c.hooks.OnCall != nil {
|
||||||
|
call.request.OnPrepared = func(_ context.Context, details pk.PreparedRun) {
|
||||||
|
metadata := promptMetadata(details)
|
||||||
|
var snapshot *structuredoutput.PreparedSnapshot
|
||||||
|
if req.CaptureDiagnostics {
|
||||||
|
snapshot = &structuredoutput.PreparedSnapshot{Messages: convertMessages(details.Messages), Prompt: metadata}
|
||||||
|
}
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
prepared = snapshot
|
prepared, prompt = snapshot, metadata
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
return call, nil
|
return call, nil
|
||||||
}
|
}
|
||||||
@@ -290,11 +297,6 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
|
|
||||||
messages := convertMessages(details.Messages)
|
|
||||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
|
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
|
||||||
result := make([]structuredoutput.Message, len(messages))
|
result := make([]structuredoutput.Message, len(messages))
|
||||||
for i, message := range messages {
|
for i, message := range messages {
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
|||||||
id = prompts.PromptIDValidatorEditorialReview
|
id = prompts.PromptIDValidatorEditorialReview
|
||||||
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
|
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
|
||||||
}
|
}
|
||||||
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
return structuredoutput.Request{CaptureDiagnostics: true, PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
|
|||||||
if turns == 2 {
|
if turns == 2 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
|
||||||
})
|
})
|
||||||
if err != nil || exhausted || turns != 2 {
|
if err != nil || exhausted || turns != 2 {
|
||||||
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
|
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
|
||||||
@@ -111,7 +111,7 @@ func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
|
|||||||
if turn.Response != nil {
|
if turn.Response != nil {
|
||||||
return nil, turn.Err
|
return nil, turn.Err
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
|
||||||
}
|
}
|
||||||
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
|
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
|
||||||
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
|
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
|
||||||
@@ -172,7 +172,7 @@ func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T)
|
|||||||
result = turn.Response
|
result = turn.Response
|
||||||
return nil, turn.Err
|
return nil, turn.Err
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
|
||||||
})
|
})
|
||||||
if err != nil || exhausted || result == nil {
|
if err != nil || exhausted || result == nil {
|
||||||
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)
|
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||||
})
|
})
|
||||||
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
|
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
|
||||||
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
|
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
|
||||||
@@ -102,7 +102,7 @@ func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return nil, turn.Err
|
return nil, turn.Err
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||||
})
|
})
|
||||||
if !errors.Is(err, context.Canceled) || deliveries != 1 {
|
if !errors.Is(err, context.Canceled) || deliveries != 1 {
|
||||||
t.Fatalf("deliveries=%d err=%v", deliveries, err)
|
t.Fatalf("deliveries=%d err=%v", deliveries, err)
|
||||||
@@ -132,7 +132,7 @@ func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
|
|||||||
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
|
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||||
})
|
})
|
||||||
summary := Summarize(collector.Snapshot())
|
summary := Summarize(collector.Snapshot())
|
||||||
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
|
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
|
||||||
|
|||||||
@@ -109,6 +109,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
|
|||||||
var result map[string]any
|
var result map[string]any
|
||||||
capturing := request
|
capturing := request
|
||||||
capturing.CaptureExchange = true
|
capturing.CaptureExchange = true
|
||||||
|
capturing.CaptureDiagnostics = true
|
||||||
response, err := client.CompleteStructured(context.Background(), capturing, &result)
|
response, err := client.CompleteStructured(context.Background(), capturing, &result)
|
||||||
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
|
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
|
||||||
t.Fatalf("captured final response: %+v", response.Exchange)
|
t.Fatalf("captured final response: %+v", response.Exchange)
|
||||||
@@ -261,6 +262,7 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
req.CaptureDiagnostics = true
|
||||||
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
response, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -121,3 +121,57 @@ func ExecutionRecoveries(counts ExecutionCounts) []RecoveryReason {
|
|||||||
}
|
}
|
||||||
return reasons
|
return reasons
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WorkTotals retains the facts shared by the CLI summary and ordinary reports.
|
||||||
|
type WorkTotals struct {
|
||||||
|
Total int
|
||||||
|
Started int
|
||||||
|
Completed int
|
||||||
|
WithErrors int
|
||||||
|
Corrections int
|
||||||
|
ModelOperations int
|
||||||
|
ProposalResponses int
|
||||||
|
Recoveries int
|
||||||
|
FilteringResponses int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *WorkTotals) Record(state WorkState, operations []Operation) {
|
||||||
|
if state != WorkNotStarted {
|
||||||
|
t.Started++
|
||||||
|
}
|
||||||
|
if state == WorkCompleted {
|
||||||
|
t.Completed++
|
||||||
|
}
|
||||||
|
hasError := false
|
||||||
|
for _, operation := range operations {
|
||||||
|
if operation.Reference.Kind != OperationValidationInput {
|
||||||
|
t.ModelOperations++
|
||||||
|
}
|
||||||
|
if operation.Succeeded {
|
||||||
|
if operation.Reference.Kind == OperationProposal {
|
||||||
|
t.ProposalResponses++
|
||||||
|
if len(operation.Filtered) > 0 {
|
||||||
|
t.FilteringResponses++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(operation.RecoveryReasons) > 0 {
|
||||||
|
t.Recoveries++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hasError = hasError || operation.FailureClass == FailureRecoverable
|
||||||
|
}
|
||||||
|
if hasError {
|
||||||
|
t.WithErrors++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *WorkTotals) Add(module WorkTotals) {
|
||||||
|
t.Started += module.Started
|
||||||
|
t.Completed += module.Completed
|
||||||
|
t.WithErrors += module.WithErrors
|
||||||
|
t.Corrections += module.Corrections
|
||||||
|
t.ModelOperations += module.ModelOperations
|
||||||
|
t.ProposalResponses += module.ProposalResponses
|
||||||
|
t.Recoveries += module.Recoveries
|
||||||
|
t.FilteringResponses += module.FilteringResponses
|
||||||
|
}
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
|
|||||||
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
|
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
|
||||||
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
|
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
|
||||||
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
|
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true,
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, Evidence: &runner.EvidenceSelection{Reports: true, Debug: true},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, origin) || client.calls != 3 {
|
if !errors.Is(err, origin) || client.calls != 3 {
|
||||||
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)
|
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)
|
||||||
|
|||||||
@@ -66,59 +66,35 @@ func Build(input BuildInput) reporting.Snapshot {
|
|||||||
return units[i].Section.Index < units[j].Section.Index
|
return units[i].Section.Index < units[j].Section.Index
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
total, started, completed, interrupted, notStarted, withErrors := len(units), 0, 0, 0, 0, 0
|
var measured outcomes.WorkTotals
|
||||||
corrections, llmOperations, proposalResponses, recoveryCount, filteringResponses := 0, 0, 0, 0, 0
|
if input.RunOutput.Totals != nil {
|
||||||
|
measured = *input.RunOutput.Totals
|
||||||
|
} else {
|
||||||
|
measured.Total = len(units)
|
||||||
|
for _, unit := range units {
|
||||||
|
measured.Record(unit.State, unit.Operations)
|
||||||
|
measured.Corrections += unit.CorrectionsApplied
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, unit := range units {
|
for _, unit := range units {
|
||||||
if unit.State == outcomes.WorkNotStarted {
|
if !details {
|
||||||
notStarted++
|
break
|
||||||
} else {
|
|
||||||
started++
|
|
||||||
}
|
|
||||||
if unit.State == outcomes.WorkCompleted {
|
|
||||||
completed++
|
|
||||||
}
|
|
||||||
if unit.State == outcomes.WorkInterrupted {
|
|
||||||
interrupted++
|
|
||||||
}
|
|
||||||
corrections += unit.CorrectionsApplied
|
|
||||||
operations := unit.Operations
|
|
||||||
if details {
|
|
||||||
operations = append([]outcomes.Operation(nil), operations...)
|
|
||||||
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
|
|
||||||
}
|
}
|
||||||
|
operations := append([]outcomes.Operation(nil), unit.Operations...)
|
||||||
|
sort.SliceStable(operations, func(i, j int) bool { return operationLess(operations[i], operations[j]) })
|
||||||
var failures []reporting.Failure
|
var failures []reporting.Failure
|
||||||
unitHasError := false
|
unitHasError := false
|
||||||
unitRecoveries := 0
|
unitRecoveries := 0
|
||||||
for _, operation := range operations {
|
for _, operation := range operations {
|
||||||
if operation.Reference.Kind != outcomes.OperationValidationInput {
|
|
||||||
llmOperations++
|
|
||||||
}
|
|
||||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
|
||||||
proposalResponses++
|
|
||||||
}
|
|
||||||
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
|
if operation.Succeeded && len(operation.RecoveryReasons) > 0 {
|
||||||
recoveryCount++
|
|
||||||
unitRecoveries++
|
unitRecoveries++
|
||||||
if len(operation.Filtered) > 0 && operation.Reference.Kind == outcomes.OperationProposal {
|
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
||||||
filteringResponses++
|
|
||||||
}
|
|
||||||
if details {
|
|
||||||
snapshot.Recoveries = append(snapshot.Recoveries, mapRecovery(unit, operation))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||||
unitHasError = true
|
unitHasError = true
|
||||||
if details {
|
failures = append(failures, mapFailure(operation))
|
||||||
failures = append(failures, mapFailure(operation))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if unitHasError {
|
|
||||||
withErrors++
|
|
||||||
}
|
|
||||||
if !details {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
row := reporting.WorkUnit{StageIndex: unit.ModuleIndex, StageID: unit.StageID, ModuleKey: unit.ModuleKey, ModuleInstance: unit.ModuleInstance,
|
||||||
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
ChunkIndex: unit.Section.Index, StartSegmentID: unit.Section.StartSegmentID, EndSegmentID: unit.Section.EndSegmentID,
|
||||||
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
|
State: string(unit.State), CorrectionsApplied: unit.CorrectionsApplied, RecoveryCount: unitRecoveries}
|
||||||
@@ -153,12 +129,13 @@ func Build(input BuildInput) reporting.Snapshot {
|
|||||||
return a.ProposalIndex < b.ProposalIndex
|
return a.ProposalIndex < b.ProposalIndex
|
||||||
})
|
})
|
||||||
runErrors := len(failures)
|
runErrors := len(failures)
|
||||||
|
interrupted, notStarted := measured.Started-measured.Completed, measured.Total-measured.Started
|
||||||
snapshot.Summary = reporting.Summary{
|
snapshot.Summary = reporting.Summary{
|
||||||
WorkUnitsTotal: &total, WorkUnitsStarted: &started, WorkUnitsCompleted: &completed,
|
WorkUnitsTotal: &measured.Total, WorkUnitsStarted: &measured.Started, WorkUnitsCompleted: &measured.Completed,
|
||||||
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &withErrors,
|
WorkUnitsInterrupted: &interrupted, WorkUnitsNotStarted: ¬Started, WorkUnitsWithErrors: &measured.WithErrors,
|
||||||
RunErrors: &runErrors, CorrectionsApplied: &corrections, LLMOperations: &llmOperations,
|
RunErrors: &runErrors, CorrectionsApplied: &measured.Corrections, LLMOperations: &measured.ModelOperations,
|
||||||
ProposalResponses: &proposalResponses, Recoveries: &recoveryCount,
|
ProposalResponses: &measured.ProposalResponses, Recoveries: &measured.Recoveries,
|
||||||
ProposalResponsesWithFiltering: &filteringResponses,
|
ProposalResponsesWithFiltering: &measured.FilteringResponses,
|
||||||
}
|
}
|
||||||
} else if len(failures) > 0 {
|
} else if len(failures) > 0 {
|
||||||
runErrors := len(failures)
|
runErrors := len(failures)
|
||||||
|
|||||||
@@ -82,9 +82,10 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
|||||||
)
|
)
|
||||||
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
|
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)}
|
||||||
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
completion, callErr = req.LLMClient.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||||
CaptureExchange: req.CaptureExchange,
|
CaptureExchange: req.CaptureExchange,
|
||||||
PromptID: req.PromptID,
|
CaptureDiagnostics: req.Interactions != nil,
|
||||||
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
PromptID: req.PromptID,
|
||||||
|
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
||||||
Context: coordinates,
|
Context: coordinates,
|
||||||
}, &response)
|
}, &response)
|
||||||
sectionIndex := 0
|
sectionIndex := 0
|
||||||
|
|||||||
@@ -207,7 +207,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
|
|||||||
cfg.Chunking.MaxSectionTokens = 3
|
cfg.Chunking.MaxSectionTokens = 3
|
||||||
cfg.Chunking.MinSectionTokens = 0
|
cfg.Chunking.MinSectionTokens = 0
|
||||||
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||||
Config: &cfg, CaptureCandidates: true,
|
Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||||||
})
|
})
|
||||||
|
|||||||
94
internal/framework/runner/evidence_test.go
Normal file
94
internal/framework/runner/evidence_test.go
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||||
|
)
|
||||||
|
|
||||||
|
func evidenceFixture(size int) (*Runner, RunInput) {
|
||||||
|
cfg := runnerTestConfig()
|
||||||
|
cfg.Chunking.MaxSectionTokens = 1
|
||||||
|
cfg.Chunking.MinSectionTokens = 0
|
||||||
|
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
|
||||||
|
for i := range transcript.Segments {
|
||||||
|
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
|
||||||
|
}
|
||||||
|
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
|
||||||
|
var decisions []validators.Decision
|
||||||
|
for _, candidate := range req.CandidateProposal {
|
||||||
|
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
|
||||||
|
}
|
||||||
|
return validators.Result{Decisions: decisions}, nil
|
||||||
|
}}
|
||||||
|
module := outcomeModule{
|
||||||
|
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
|
||||||
|
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
|
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
|
||||||
|
category := req.Section.Index % 4
|
||||||
|
if category == 0 {
|
||||||
|
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
|
||||||
|
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
|
||||||
|
}
|
||||||
|
operation.Succeeded = true
|
||||||
|
original := "word"
|
||||||
|
if category == 1 {
|
||||||
|
original = "absent"
|
||||||
|
}
|
||||||
|
if category == 2 {
|
||||||
|
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
|
||||||
|
}
|
||||||
|
return contracts.ProposalResult{
|
||||||
|
Operations: []outcomes.Operation{operation},
|
||||||
|
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
|
||||||
|
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
|
||||||
|
r, input := evidenceFixture(40)
|
||||||
|
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
|
||||||
|
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
|
||||||
|
var baseline *schema.Transcript
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
selection EvidenceSelection
|
||||||
|
units int
|
||||||
|
candidates int
|
||||||
|
}{
|
||||||
|
{name: "summary"},
|
||||||
|
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
|
||||||
|
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
|
||||||
|
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
input.Evidence = &tc.selection
|
||||||
|
out, err := r.Run(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out.Totals == nil || *out.Totals != want {
|
||||||
|
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
|
||||||
|
}
|
||||||
|
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
|
||||||
|
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
|
||||||
|
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
|
||||||
|
}
|
||||||
|
if baseline == nil {
|
||||||
|
baseline = out.FinalTranscript
|
||||||
|
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
|
||||||
|
t.Fatal("output selection changed final corrections")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -161,7 +161,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
|||||||
}
|
}
|
||||||
|
|
||||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||||
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, CaptureDiagnostics: debug, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
||||||
}
|
}
|
||||||
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||||
if turn.Response == nil {
|
if turn.Response == nil {
|
||||||
|
|||||||
@@ -45,8 +45,6 @@ type ModuleResult struct {
|
|||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
ProposalCount int `json:"proposal_count"`
|
|
||||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
|
||||||
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
|
||||||
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
|
||||||
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
|
||||||
@@ -95,13 +93,32 @@ type ValidatorRejectedChange struct {
|
|||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EvidenceSelection selects the detailed views consumed by the invocation.
|
||||||
|
type EvidenceSelection struct {
|
||||||
|
Reports bool
|
||||||
|
Debug bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type evidenceNeeds struct {
|
||||||
|
reports bool
|
||||||
|
decisions bool
|
||||||
|
candidates bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func selectEvidence(selection *EvidenceSelection) evidenceNeeds {
|
||||||
|
if selection == nil {
|
||||||
|
return evidenceNeeds{reports: true, decisions: true}
|
||||||
|
}
|
||||||
|
return evidenceNeeds{reports: selection.Reports, decisions: selection.Debug, candidates: selection.Debug}
|
||||||
|
}
|
||||||
|
|
||||||
// RunInput is the deterministic runner input.
|
// RunInput is the deterministic runner input.
|
||||||
type RunInput struct {
|
type RunInput struct {
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Transcript *schema.Transcript
|
Transcript *schema.Transcript
|
||||||
ModuleSpecs []contracts.ModuleRunSpec
|
ModuleSpecs []contracts.ModuleRunSpec
|
||||||
Interactions structuredoutput.InteractionSink
|
Interactions structuredoutput.InteractionSink
|
||||||
CaptureCandidates bool
|
Evidence *EvidenceSelection
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunOutput is the deterministic runner output.
|
// RunOutput is the deterministic runner output.
|
||||||
@@ -109,6 +126,7 @@ type RunOutput struct {
|
|||||||
FinalTranscript *schema.Transcript `json:"-"`
|
FinalTranscript *schema.Transcript `json:"-"`
|
||||||
ModuleResults []ModuleResult `json:"module_results"`
|
ModuleResults []ModuleResult `json:"module_results"`
|
||||||
Sections []contracts.SectionMetadata `json:"sections"`
|
Sections []contracts.SectionMetadata `json:"sections"`
|
||||||
|
Totals *outcomes.WorkTotals `json:"-"`
|
||||||
InventoryAvailable bool `json:"-"`
|
InventoryAvailable bool `json:"-"`
|
||||||
WorkUnits []WorkUnitOutcome `json:"work_units"`
|
WorkUnits []WorkUnitOutcome `json:"work_units"`
|
||||||
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
|
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
|
||||||
@@ -132,54 +150,70 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
|
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
|
||||||
}
|
}
|
||||||
inventory := make([]contracts.SectionMetadata, len(sections))
|
evidence := selectEvidence(input.Evidence)
|
||||||
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
|
totals := outcomes.WorkTotals{Total: len(input.ModuleSpecs) * len(sections)}
|
||||||
|
var inventory []contracts.SectionMetadata
|
||||||
|
var workUnits []WorkUnitOutcome
|
||||||
var candidates []proposals.EnrichedCorrectionProposal
|
var candidates []proposals.EnrichedCorrectionProposal
|
||||||
for i, section := range sections {
|
if evidence.reports {
|
||||||
inventory[i] = contracts.SectionMetadataFromSection(section)
|
inventory = make([]contracts.SectionMetadata, len(sections))
|
||||||
}
|
for i, section := range sections {
|
||||||
for moduleIndex, spec := range input.ModuleSpecs {
|
inventory[i] = contracts.SectionMetadataFromSection(section)
|
||||||
for _, section := range inventory {
|
}
|
||||||
workUnits = append(workUnits, WorkUnitOutcome{
|
workUnits = make([]WorkUnitOutcome, 0, totals.Total)
|
||||||
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
|
for moduleIndex, spec := range input.ModuleSpecs {
|
||||||
ModuleInstance: spec.InstanceName, Section: section,
|
for _, section := range inventory {
|
||||||
State: outcomes.WorkNotStarted,
|
workUnits = append(workUnits, newWorkUnit(spec, section, moduleIndex))
|
||||||
})
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
snapshot := func() RunOutput {
|
snapshot := func() RunOutput {
|
||||||
return RunOutput{
|
return RunOutput{
|
||||||
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
|
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
|
||||||
Sections: inventory, InventoryAvailable: true, WorkUnits: workUnits, Candidates: candidates,
|
Sections: inventory, InventoryAvailable: true, Totals: &totals, WorkUnits: workUnits, Candidates: candidates,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for moduleIndex, spec := range input.ModuleSpecs {
|
for moduleIndex, spec := range input.ModuleSpecs {
|
||||||
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
|
var reportUnits []WorkUnitOutcome
|
||||||
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
|
if evidence.reports {
|
||||||
copy(units, execution.pipeline.WorkUnits)
|
reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
|
||||||
if err == nil && ctx.Err() != nil {
|
|
||||||
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
|
|
||||||
}
|
}
|
||||||
|
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
|
||||||
|
units := execution.pipeline.WorkUnits
|
||||||
if err == nil {
|
if err == nil {
|
||||||
previous := sections
|
filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
|
||||||
sections = execution.updated
|
if canceled := ctx.Err(); canceled != nil {
|
||||||
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
|
err = locateFailure(canceled, "application", failureReference(spec, nil, ""), false, nil)
|
||||||
|
} else {
|
||||||
|
execution.pipeline.recordApplyFiltering(filters, evidence.reports)
|
||||||
|
sections = execution.updated
|
||||||
|
execution.pipeline.Totals.Started = len(sections)
|
||||||
|
execution.pipeline.Totals.Completed = len(sections)
|
||||||
|
execution.pipeline.Totals.Corrections = len(execution.applied)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var failure *ExecutionFailure
|
var failure *ExecutionFailure
|
||||||
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
|
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
|
||||||
appliedIndices := make(map[int]struct{}, len(execution.applied))
|
if applicationInterrupted {
|
||||||
for _, change := range execution.applied {
|
execution.pipeline.Totals.Completed = 0
|
||||||
appliedIndices[change.ProposalIndex] = struct{}{}
|
}
|
||||||
|
totals.Add(execution.pipeline.Totals)
|
||||||
|
|
||||||
|
appliedIndices := make(map[int]struct{})
|
||||||
|
if evidence.reports || evidence.candidates {
|
||||||
|
for _, change := range execution.applied {
|
||||||
|
appliedIndices[change.ProposalIndex] = struct{}{}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
var operations []outcomes.Operation
|
|
||||||
for i := range units {
|
for i := range units {
|
||||||
unit := &units[i]
|
unit := &units[i]
|
||||||
unit.ModuleIndex = moduleIndex
|
unit.ModuleIndex = moduleIndex
|
||||||
if err == nil {
|
if err == nil {
|
||||||
unit.State = outcomes.WorkCompleted
|
unit.State = outcomes.WorkCompleted
|
||||||
unit.CorrectionsApplied = execution.appliedBySection[i]
|
if evidence.reports {
|
||||||
|
unit.CorrectionsApplied = execution.appliedBySection[unit.Section.Index]
|
||||||
|
}
|
||||||
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
|
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
|
||||||
unit.State = outcomes.WorkInterrupted
|
unit.State = outcomes.WorkInterrupted
|
||||||
}
|
}
|
||||||
@@ -196,16 +230,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
branch.Outcome = "approved_skipped"
|
branch.Outcome = "approved_skipped"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
operations = append(operations, unit.Operations...)
|
|
||||||
}
|
}
|
||||||
|
if !evidence.reports && evidence.candidates {
|
||||||
if input.CaptureCandidates {
|
workUnits = append(workUnits, units...)
|
||||||
candidates = append(candidates, execution.pipeline.Candidates...)
|
|
||||||
}
|
}
|
||||||
|
candidates = append(candidates, execution.pipeline.Candidates...)
|
||||||
result := ModuleResult{
|
result := ModuleResult{
|
||||||
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
|
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
|
||||||
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
|
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
|
||||||
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
|
|
||||||
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
|
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -213,8 +245,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
result.ErrorMessage = err.Error()
|
result.ErrorMessage = err.Error()
|
||||||
} else {
|
} else {
|
||||||
result.Status = ModuleStatusSuccess
|
result.Status = ModuleStatusSuccess
|
||||||
result.AppliedChanges = execution.applied
|
if evidence.decisions {
|
||||||
result.SkippedChanges = execution.skipped
|
result.AppliedChanges = execution.applied
|
||||||
|
result.SkippedChanges = execution.skipped
|
||||||
|
}
|
||||||
}
|
}
|
||||||
results = append(results, result)
|
results = append(results, result)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -237,7 +271,7 @@ type moduleExecution struct {
|
|||||||
appliedBySection []int
|
appliedBySection []int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
|
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
|
||||||
var execution moduleExecution
|
var execution moduleExecution
|
||||||
origin := failureReference(spec, nil, "")
|
origin := failureReference(spec, nil, "")
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
@@ -253,6 +287,8 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
|
|||||||
}
|
}
|
||||||
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
|
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
|
||||||
Module: module,
|
Module: module,
|
||||||
|
Evidence: &evidence,
|
||||||
|
ReportUnits: reportUnits,
|
||||||
Spec: spec,
|
Spec: spec,
|
||||||
Policy: execution.policy,
|
Policy: execution.policy,
|
||||||
Sections: sections,
|
Sections: sections,
|
||||||
@@ -323,6 +359,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
|||||||
}
|
}
|
||||||
|
|
||||||
type collectSectionProposalsInput struct {
|
type collectSectionProposalsInput struct {
|
||||||
|
Evidence *evidenceNeeds
|
||||||
|
ReportUnits []WorkUnitOutcome
|
||||||
Interactions structuredoutput.InteractionSink
|
Interactions structuredoutput.InteractionSink
|
||||||
Module contracts.TranscriptModule
|
Module contracts.TranscriptModule
|
||||||
Spec contracts.ModuleRunSpec
|
Spec contracts.ModuleRunSpec
|
||||||
@@ -356,16 +394,18 @@ type sectionValidationResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type sectionWork struct {
|
type sectionWork struct {
|
||||||
|
unit WorkUnitOutcome
|
||||||
candidates []proposals.EnrichedCorrectionProposal
|
candidates []proposals.EnrichedCorrectionProposal
|
||||||
validation *sectionValidationResult
|
validation *sectionValidationResult
|
||||||
finished bool
|
finished bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type modulePipelineResult struct {
|
type modulePipelineResult struct {
|
||||||
ProposalCount int
|
Totals outcomes.WorkTotals
|
||||||
|
proposalFacts map[int]proposalRecovery
|
||||||
|
revised map[int]struct{}
|
||||||
Candidates []proposals.EnrichedCorrectionProposal
|
Candidates []proposals.EnrichedCorrectionProposal
|
||||||
Approved []proposals.EnrichedCorrectionProposal
|
Approved []proposals.EnrichedCorrectionProposal
|
||||||
Operations []outcomes.Operation
|
|
||||||
WorkUnits []WorkUnitOutcome
|
WorkUnits []WorkUnitOutcome
|
||||||
ValidatorDecisions []ValidatorDecisionRecord
|
ValidatorDecisions []ValidatorDecisionRecord
|
||||||
ValidatorRejected []ValidatorRejectedChange
|
ValidatorRejected []ValidatorRejectedChange
|
||||||
@@ -410,17 +450,16 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
|
|||||||
}
|
}
|
||||||
|
|
||||||
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
|
||||||
out := modulePipelineResult{
|
var out modulePipelineResult
|
||||||
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
|
evidence := selectEvidence(nil)
|
||||||
Operations: make([]outcomes.Operation, 0),
|
if input.Evidence != nil {
|
||||||
WorkUnits: make([]WorkUnitOutcome, len(input.Sections)),
|
evidence = *input.Evidence
|
||||||
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
|
|
||||||
ValidatorRejected: make([]ValidatorRejectedChange, 0),
|
|
||||||
}
|
}
|
||||||
for i, section := range input.Sections {
|
out.WorkUnits = input.ReportUnits
|
||||||
out.WorkUnits[i] = WorkUnitOutcome{
|
if evidence.reports && out.WorkUnits == nil {
|
||||||
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID,
|
out.WorkUnits = make([]WorkUnitOutcome, len(input.Sections))
|
||||||
Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted,
|
for i, section := range input.Sections {
|
||||||
|
out.WorkUnits[i] = newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(section), 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -487,17 +526,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
return sectionEnriched
|
return sectionEnriched
|
||||||
}
|
}
|
||||||
|
|
||||||
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
|
recordCandidates := func(unit *WorkUnitOutcome, enriched []proposals.EnrichedCorrectionProposal) {
|
||||||
unit := &out.WorkUnits[sectionPos]
|
|
||||||
unit.ProposalIndexOffset = nextProposalIndex
|
unit.ProposalIndexOffset = nextProposalIndex
|
||||||
for _, candidate := range enriched {
|
if evidence.candidates {
|
||||||
candidate.ProposalIndex += nextProposalIndex
|
for _, candidate := range enriched {
|
||||||
out.Candidates = append(out.Candidates, candidate)
|
candidate.ProposalIndex += nextProposalIndex
|
||||||
|
out.Candidates = append(out.Candidates, candidate)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out.ProposalCount += len(enriched)
|
|
||||||
for operationIndex := range unit.Operations {
|
for operationIndex := range unit.Operations {
|
||||||
operation := &unit.Operations[operationIndex]
|
operation := &unit.Operations[operationIndex]
|
||||||
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
if evidence.reports && operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||||
for _, proposal := range enriched {
|
for _, proposal := range enriched {
|
||||||
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
|
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
|
||||||
}
|
}
|
||||||
@@ -516,16 +555,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
interactions = capture
|
interactions = capture
|
||||||
}
|
}
|
||||||
validationInput := validateSectionCandidatesInput{
|
validationInput := validateSectionCandidatesInput{
|
||||||
Spec: input.Spec,
|
Spec: input.Spec,
|
||||||
Policy: input.Policy,
|
SuppressDecisionHistory: !evidence.decisions,
|
||||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
Policy: input.Policy,
|
||||||
SectionIndex: §ionMetadata.Index,
|
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||||
Validators: validatorsOrdered,
|
SectionIndex: §ionMetadata.Index,
|
||||||
SectionEnriched: enriched,
|
Validators: validatorsOrdered,
|
||||||
Exchange: sectionResult.proposal.Exchange,
|
SectionEnriched: enriched,
|
||||||
ValidationLLMClient: input.ValidationClient,
|
Exchange: sectionResult.proposal.Exchange,
|
||||||
GlossaryView: input.GlossaryView,
|
ValidationLLMClient: input.ValidationClient,
|
||||||
Interactions: interactions,
|
GlossaryView: input.GlossaryView,
|
||||||
|
Interactions: interactions,
|
||||||
}
|
}
|
||||||
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
|
||||||
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
|
||||||
@@ -555,8 +595,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
failure: failure,
|
failure: failure,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
recordValidation := func(res sectionValidationResult) {
|
recordValidation := func(unit *WorkUnitOutcome, res sectionValidationResult) {
|
||||||
unit := &out.WorkUnits[res.sectionPos]
|
|
||||||
unit.Operations = append(unit.Operations, res.operations...)
|
unit.Operations = append(unit.Operations, res.operations...)
|
||||||
unit.Repairs = res.repairs
|
unit.Repairs = res.repairs
|
||||||
mergeFilteredProposals(unit, res.filtered)
|
mergeFilteredProposals(unit, res.filtered)
|
||||||
@@ -567,18 +606,30 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
unit.State = outcomes.WorkInterrupted
|
unit.State = outcomes.WorkInterrupted
|
||||||
}
|
}
|
||||||
out.Approved = append(out.Approved, res.approved...)
|
out.Approved = append(out.Approved, res.approved...)
|
||||||
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
if evidence.decisions {
|
||||||
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
|
||||||
|
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
|
||||||
|
}
|
||||||
|
if len(res.approved) > 0 {
|
||||||
|
out.rememberProposalRecovery(unit)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nextAdmission, active := 0, 0
|
nextAdmission, active := 0, 0
|
||||||
for {
|
for {
|
||||||
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
|
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
|
||||||
offset := nextProposalIndex
|
offset := nextProposalIndex
|
||||||
recordCandidates(nextSectionToRecord, work.candidates)
|
recordCandidates(&work.unit, work.candidates)
|
||||||
if work.validation != nil {
|
if work.validation != nil {
|
||||||
work.validation.assignProposalIndices(offset)
|
work.validation.assignProposalIndices(offset)
|
||||||
recordValidation(*work.validation)
|
recordValidation(&work.unit, *work.validation)
|
||||||
|
}
|
||||||
|
out.Totals.Record(work.unit.State, work.unit.Operations)
|
||||||
|
if evidence.reports {
|
||||||
|
out.WorkUnits[nextSectionToRecord] = work.unit
|
||||||
|
} else if evidence.candidates && len(work.unit.Repairs) > 0 {
|
||||||
|
work.unit.Operations = nil
|
||||||
|
out.WorkUnits = append(out.WorkUnits, work.unit)
|
||||||
}
|
}
|
||||||
delete(pending, nextSectionToRecord)
|
delete(pending, nextSectionToRecord)
|
||||||
nextSectionToRecord++
|
nextSectionToRecord++
|
||||||
@@ -586,7 +637,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
|
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
|
||||||
sectionPos := nextAdmission
|
sectionPos := nextAdmission
|
||||||
nextAdmission++
|
nextAdmission++
|
||||||
pending[sectionPos] = §ionWork{}
|
pending[sectionPos] = §ionWork{unit: newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(input.Sections[sectionPos]), 0)}
|
||||||
active++
|
active++
|
||||||
workers.Go(func() {
|
workers.Go(func() {
|
||||||
proposed := proposeSection(runCtx, input, sectionPos)
|
proposed := proposeSection(runCtx, input, sectionPos)
|
||||||
@@ -600,13 +651,13 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
select {
|
select {
|
||||||
case result := <-proposalResults:
|
case result := <-proposalResults:
|
||||||
active--
|
active--
|
||||||
unit := &out.WorkUnits[result.sectionPos]
|
work := pending[result.sectionPos]
|
||||||
|
unit := &work.unit
|
||||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||||
if result.started {
|
if result.started {
|
||||||
unit.State = outcomes.WorkInterrupted
|
unit.State = outcomes.WorkInterrupted
|
||||||
}
|
}
|
||||||
unit.Failure = result.failure
|
unit.Failure = result.failure
|
||||||
work := pending[result.sectionPos]
|
|
||||||
if result.started && result.failure == nil {
|
if result.started && result.failure == nil {
|
||||||
work.candidates = makeCandidates(result)
|
work.candidates = makeCandidates(result)
|
||||||
if runCtx.Err() == nil {
|
if runCtx.Err() == nil {
|
||||||
@@ -633,10 +684,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
|
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
|
||||||
})
|
})
|
||||||
|
|
||||||
for _, unit := range out.WorkUnits {
|
|
||||||
out.Operations = append(out.Operations, unit.Operations...)
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
|
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
|
||||||
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
|
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
|
||||||
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
|
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
|
||||||
@@ -668,19 +715,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
}
|
}
|
||||||
|
|
||||||
type validateSectionCandidatesInput struct {
|
type validateSectionCandidatesInput struct {
|
||||||
Section *validators.SectionLookup
|
SuppressDecisionHistory bool
|
||||||
RepairProposalIndex *int
|
Section *validators.SectionLookup
|
||||||
RepairAttempt *int
|
RepairProposalIndex *int
|
||||||
Exchange *structuredoutput.Exchange
|
RepairAttempt *int
|
||||||
Spec contracts.ModuleRunSpec
|
Exchange *structuredoutput.Exchange
|
||||||
Policy proposals.ReplacementPolicy
|
Spec contracts.ModuleRunSpec
|
||||||
WorkingTranscript *schema.Transcript
|
Policy proposals.ReplacementPolicy
|
||||||
SectionIndex *int
|
WorkingTranscript *schema.Transcript
|
||||||
Validators []validators.Validator
|
SectionIndex *int
|
||||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
Validators []validators.Validator
|
||||||
ValidationLLMClient contracts.StructuredLLMClient
|
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||||
Interactions structuredoutput.InteractionSink
|
ValidationLLMClient contracts.StructuredLLMClient
|
||||||
GlossaryView *validators.GlossaryView
|
Interactions structuredoutput.InteractionSink
|
||||||
|
GlossaryView *validators.GlossaryView
|
||||||
}
|
}
|
||||||
|
|
||||||
type validateSectionCandidatesResult struct {
|
type validateSectionCandidatesResult struct {
|
||||||
@@ -781,16 +829,18 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
noEffect[d.ProposalIndex] = true
|
noEffect[d.ProposalIndex] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
decisions = append(decisions, ValidatorDecisionRecord{
|
if !input.SuppressDecisionHistory {
|
||||||
ValidatorName: validator.Name(),
|
decisions = append(decisions, ValidatorDecisionRecord{
|
||||||
ExecutionClass: class,
|
ValidatorName: validator.Name(),
|
||||||
ProposalIndex: d.ProposalIndex,
|
ExecutionClass: class,
|
||||||
Approved: approved,
|
ProposalIndex: d.ProposalIndex,
|
||||||
Disposition: disposition,
|
Approved: approved,
|
||||||
ReasonCode: d.ReasonCode,
|
Disposition: disposition,
|
||||||
Message: d.Message,
|
ReasonCode: d.ReasonCode,
|
||||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
Message: d.Message,
|
||||||
})
|
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
if approved {
|
if approved {
|
||||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
||||||
continue
|
continue
|
||||||
@@ -800,19 +850,21 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
|
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p := byIndex[d.ProposalIndex]
|
if feedback || !input.SuppressDecisionHistory {
|
||||||
rejected = append(rejected, ValidatorRejectedChange{
|
p := byIndex[d.ProposalIndex]
|
||||||
ValidatorName: validator.Name(),
|
rejected = append(rejected, ValidatorRejectedChange{
|
||||||
ProposalIndex: p.ProposalIndex,
|
ValidatorName: validator.Name(),
|
||||||
ModuleKey: p.ModuleKey,
|
ProposalIndex: p.ProposalIndex,
|
||||||
ModuleInstance: p.ModuleInstance,
|
ModuleKey: p.ModuleKey,
|
||||||
TargetSegmentID: p.TargetSegmentID,
|
ModuleInstance: p.ModuleInstance,
|
||||||
OriginalText: p.OriginalText,
|
TargetSegmentID: p.TargetSegmentID,
|
||||||
CorrectedText: p.CorrectedText,
|
OriginalText: p.OriginalText,
|
||||||
Disposition: disposition,
|
CorrectedText: p.CorrectedText,
|
||||||
ReasonCode: d.ReasonCode,
|
Disposition: disposition,
|
||||||
Message: d.Message,
|
ReasonCode: d.ReasonCode,
|
||||||
})
|
Message: d.Message,
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -877,9 +929,9 @@ func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) {
|
func findApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy) map[int][]outcomes.FilteredProposal {
|
||||||
if len(skipped) == 0 {
|
if len(skipped) == 0 {
|
||||||
return
|
return nil
|
||||||
}
|
}
|
||||||
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
|
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
|
||||||
sectionPosition := make(map[int]int, len(sections))
|
sectionPosition := make(map[int]int, len(sections))
|
||||||
@@ -916,26 +968,7 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
|
|||||||
}
|
}
|
||||||
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
|
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
|
||||||
}
|
}
|
||||||
for pos, items := range filtered {
|
return filtered
|
||||||
unit := &units[pos]
|
|
||||||
revisions := make(map[int]*outcomes.Operation)
|
|
||||||
for i := range unit.Operations {
|
|
||||||
op := &unit.Operations[i]
|
|
||||||
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil &&
|
|
||||||
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
|
|
||||||
revisions[*op.Reference.ProposalIndex] = op
|
|
||||||
}
|
|
||||||
}
|
|
||||||
original := items[:0]
|
|
||||||
for _, item := range items {
|
|
||||||
if op := revisions[item.ProposalIndex]; op != nil {
|
|
||||||
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
|
|
||||||
} else {
|
|
||||||
original = append(original, item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
mergeFilteredProposals(unit, original)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
|
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
|
||||||
|
|||||||
@@ -173,7 +173,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
|
|||||||
if len(without.Candidates) != 0 {
|
if len(without.Candidates) != 0 {
|
||||||
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
|
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
|
||||||
}
|
}
|
||||||
input.CaptureCandidates = true
|
input.Evidence = &EvidenceSelection{Reports: true, Debug: true}
|
||||||
with, err := r.Run(context.Background(), input)
|
with, err := r.Run(context.Background(), input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -844,7 +844,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true})
|
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -973,8 +973,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
|
|||||||
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
|
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
|
||||||
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
|
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
|
||||||
}
|
}
|
||||||
if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
|
if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
|
||||||
t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations)
|
t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -218,7 +218,7 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
|||||||
cfg.Chunking.MaxSectionTokens = 1
|
cfg.Chunking.MaxSectionTokens = 1
|
||||||
cfg.Chunking.MinSectionTokens = 0
|
cfg.Chunking.MinSectionTokens = 0
|
||||||
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||||
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
|
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
||||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||||
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
||||||
|
|||||||
78
internal/framework/runner/work_totals.go
Normal file
78
internal/framework/runner/work_totals.go
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newWorkUnit(spec contracts.ModuleRunSpec, section contracts.SectionMetadata, moduleIndex int) WorkUnitOutcome {
|
||||||
|
return WorkUnitOutcome{
|
||||||
|
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
|
||||||
|
ModuleInstance: spec.InstanceName, Section: section, State: outcomes.WorkNotStarted,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type proposalRecovery struct {
|
||||||
|
recovered bool
|
||||||
|
filtered bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Application can discover additional defects. Keep only the proposal operation
|
||||||
|
// flags needed to update totals and the identities of approved revisions.
|
||||||
|
func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) {
|
||||||
|
for _, operation := range unit.Operations {
|
||||||
|
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
|
||||||
|
if p.proposalFacts == nil {
|
||||||
|
p.proposalFacts = make(map[int]proposalRecovery)
|
||||||
|
}
|
||||||
|
p.proposalFacts[unit.Section.Index] = proposalRecovery{
|
||||||
|
recovered: len(operation.RecoveryReasons) > 0, filtered: len(operation.Filtered) > 0,
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, repair := range unit.Repairs {
|
||||||
|
if repair.Outcome == "approved" {
|
||||||
|
if p.revised == nil {
|
||||||
|
p.revised = make(map[int]struct{})
|
||||||
|
}
|
||||||
|
p.revised[repair.ProposalIndex] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) {
|
||||||
|
for pos, items := range filters {
|
||||||
|
original := items[:0]
|
||||||
|
for _, item := range items {
|
||||||
|
if _, repaired := p.revised[item.ProposalIndex]; !repaired {
|
||||||
|
original = append(original, item)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if reports {
|
||||||
|
operations := p.WorkUnits[pos].Operations
|
||||||
|
for i := range operations {
|
||||||
|
op := &operations[i]
|
||||||
|
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
|
||||||
|
*op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
|
||||||
|
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if facts, ok := p.proposalFacts[pos]; ok && len(original) != 0 {
|
||||||
|
if !facts.recovered {
|
||||||
|
p.Totals.Recoveries++
|
||||||
|
}
|
||||||
|
if !facts.filtered {
|
||||||
|
p.Totals.FilteringResponses++
|
||||||
|
}
|
||||||
|
if reports {
|
||||||
|
mergeFilteredProposals(&p.WorkUnits[pos], original)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,11 +26,12 @@ type DiagnosticContext struct {
|
|||||||
// Request names an embedded prompt and response schema. Profile selection and
|
// Request names an embedded prompt and response schema. Profile selection and
|
||||||
// credentials are supplied by the role-bound runtime client.
|
// credentials are supplied by the role-bound runtime client.
|
||||||
type Request struct {
|
type Request struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
Variables map[string]string `json:"variables"`
|
Variables map[string]string `json:"variables"`
|
||||||
Schema responseschema.Key `json:"schema"`
|
Schema responseschema.Key `json:"schema"`
|
||||||
Context DiagnosticContext `json:"context"`
|
Context DiagnosticContext `json:"context"`
|
||||||
CaptureExchange bool `json:"-"`
|
CaptureExchange bool `json:"-"`
|
||||||
|
CaptureDiagnostics bool `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptMetadata is the execution snapshot returned by the runtime when available.
|
// PromptMetadata is the execution snapshot returned by the runtime when available.
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
||||||
PromptID: promptMetadata.PromptID,
|
PromptID: promptMetadata.PromptID,
|
||||||
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
||||||
Context: coordinates,
|
Context: coordinates, CaptureDiagnostics: req.Interactions != nil,
|
||||||
}, &response)
|
}, &response)
|
||||||
batchIndex := batch.BatchIndex
|
batchIndex := batch.BatchIndex
|
||||||
operation := outcomes.Operation{
|
operation := outcomes.Operation{
|
||||||
|
|||||||
Reference in New Issue
Block a user