Retain repair diagnostics and correction lineage

This commit is contained in:
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions

View File

@@ -101,6 +101,9 @@ holds logical calls and PromptKit execution events. Interaction request/response
artifacts and input snapshots are also debug-only. Records carry zero-based artifacts and input snapshots are also debug-only. Records carry zero-based
`stage_index` and optional `stage_id` where an entry is identified. `stage_index` and optional `stage_id` where an entry is identified.
These files are diagnostic evidence, not an alternate ordinary report schema. These files are diagnostic evidence, not an alternate ordinary report schema.
For a repaired proposal, one ledger row preserves the original proposed edit,
the final applied segment and span when applicable, and repair attempts with
revision decisions and interaction artifact paths.
Each candidate-backed ledger entry includes `proposed_corrected_text`, the Each candidate-backed ledger entry includes `proposed_corrected_text`, the
literal replacement span, even when it is `""` for a deletion. Applied entries literal replacement span, even when it is `""` for a deletion. Applied entries
also include `applied_corrected_text`, likewise `""` for an applied deletion. also include `applied_corrected_text`, likewise `""` for an applied deletion.

View File

@@ -37,7 +37,9 @@ under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
when debug is off. With debug enabled, it captures source and normalized when debug is off. With debug enabled, it captures source and normalized
snapshots, invocation, effective settings, correction ledger, LLM execution snapshots, invocation, effective settings, correction ledger, LLM execution
observations, and model interactions. Candidate capture includes proposals later observations, and model interactions. Candidate capture includes proposals later
rejected or interrupted. Ledger mapping preserves empty proposed and applied rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable; their retained content
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
replacements and distinguishes unavailable candidates from interrupted work. replacements and distinguishes unavailable candidates from interrupted work.
A concurrency-safe diagnostic callback reports optional write failures without A concurrency-safe diagnostic callback reports optional write failures without

View File

@@ -322,6 +322,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
## Stage 8 — Retain optional repair evidence and preserve ledger lineage ## Stage 8 — Retain optional repair evidence and preserve ledger lineage
**Completed.**
1. Extend the current interaction writers and logical execution observation 1. Extend the current interaction writers and logical execution observation
data with a repair role, original proposal index, and one-based semantic data with a repair role, original proposal index, and one-based semantic
attempt number. Use generated stage instance, section, proposal, and attempt number. Use generated stage instance, section, proposal, and

View File

@@ -53,13 +53,14 @@ type ExecutionObservation struct {
} }
type LogicalCall struct { type LogicalCall struct {
CallID uint64 `json:"call_id"` RequestedPromptID string `json:"requested_prompt_id,omitempty"`
Context structuredoutput.DiagnosticContext `json:"context"` CallID uint64 `json:"call_id"`
Prompt structuredoutput.PromptMetadata `json:"prompt"` Context structuredoutput.DiagnosticContext `json:"context"`
Started time.Time `json:"started"` Prompt structuredoutput.PromptMetadata `json:"prompt"`
Finished time.Time `json:"finished,omitempty"` Started time.Time `json:"started"`
WallDuration time.Duration `json:"wall_duration_ns"` Finished time.Time `json:"finished,omitempty"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"` WallDuration time.Duration `json:"wall_duration_ns"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
} }
type ObservationSnapshot struct { type ObservationSnapshot struct {
@@ -92,6 +93,14 @@ func (c *ObservationCollector) Hooks() CallHooks {
} }
func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext { func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext {
if value.ProposalIndex != nil {
index := *value.ProposalIndex
value.ProposalIndex = &index
}
if value.RepairAttempt != nil {
index := *value.RepairAttempt
value.RepairAttempt = &index
}
if value.StageIndex != nil { if value.StageIndex != nil {
index := *value.StageIndex index := *value.StageIndex
value.StageIndex = &index value.StageIndex = &index
@@ -117,7 +126,7 @@ func (c *ObservationCollector) recordCall(record structuredoutput.CallRecord) {
} }
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
call := LogicalCall{CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure} call := LogicalCall{RequestedPromptID: record.RequestedPromptID, CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure}
if !record.Finished.IsZero() { if !record.Finished.IsZero() {
call.WallDuration = record.Finished.Sub(record.Started) call.WallDuration = record.Finished.Sub(record.Started)
} }

View File

@@ -4,6 +4,7 @@ import (
"path/filepath" "path/filepath"
"sort" "sort"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -33,6 +34,9 @@ type CorrectionLedgerEntry struct {
SegmentID int `json:"segment_id,omitempty"` SegmentID int `json:"segment_id,omitempty"`
OriginalText string `json:"original_text,omitempty"` OriginalText string `json:"original_text,omitempty"`
ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"` ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"`
AppliedSegmentID *int `json:"applied_segment_id,omitempty"`
AppliedOriginalText *string `json:"applied_original_text,omitempty"`
RepairAttempts []LedgerRepairAttempt `json:"repair_attempts,omitempty"`
AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"` AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"`
ReplacementPolicy string `json:"replacement_policy,omitempty"` ReplacementPolicy string `json:"replacement_policy,omitempty"`
Disposition string `json:"disposition"` Disposition string `json:"disposition"`
@@ -42,7 +46,20 @@ type CorrectionLedgerEntry struct {
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"` LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
} }
type LedgerRepairAttempt struct {
Attempt int `json:"attempt"`
Revision *proposals.EnrichedCorrectionProposal `json:"revision,omitempty"`
FailureReason string `json:"failure_reason,omitempty"`
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type LedgerValidatorDecisionRecord struct { type LedgerValidatorDecisionRecord struct {
ArtifactPath string `json:"artifact_path,omitempty"`
ValidatorKey string `json:"validator_key"` ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"` Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"` Disposition validators.DecisionDisposition `json:"disposition"`
@@ -71,6 +88,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
policy string policy string
decisions map[int][]runner.ValidatorDecisionRecord decisions map[int][]runner.ValidatorDecisionRecord
} }
repairs := make(map[proposalIdentity]runner.RepairOutcome)
for _, unit := range runOutput.WorkUnits {
for _, repair := range unit.Repairs {
repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair
}
}
seen := make(map[proposalIdentity]bool) seen := make(map[proposalIdentity]bool)
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults)) stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
for _, module := range runOutput.ModuleResults { for _, module := range runOutput.ModuleResults {
@@ -85,6 +108,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
} }
for _, change := range module.AppliedChanges { for _, change := range module.AppliedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID, StageIndex: module.StageIndex, StageID: module.StageID,
@@ -103,6 +129,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}) })
} }
for _, change := range module.SkippedChanges { for _, change := range module.SkippedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{ entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID, StageIndex: module.StageIndex, StageID: module.StageID,
@@ -122,6 +151,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}) })
} }
for _, rejection := range module.ValidatorRejected { for _, rejection := range module.ValidatorRejected {
if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] {
continue
}
if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok {
continue
}
disposition := correctionDispositionRejected disposition := correctionDispositionRejected
if rejection.Disposition == validators.DispositionUnavailable { if rejection.Disposition == validators.DispositionUnavailable {
disposition = correctionDispositionUnavailable disposition = correctionDispositionUnavailable
@@ -156,6 +191,23 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}) })
} }
} }
for identity, repair := range repairs {
stage := stages[identity.instance]
original := repair.Original
entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)}
for i, attempt := range repair.Attempts {
item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)}
entry.RepairAttempts = append(entry.RepairAttempts, item)
}
if repair.Outcome == "applied" {
segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText
entry.AppliedSegmentID = &segment
entry.AppliedOriginalText = &before
entry.AppliedCorrectedText = &after
}
seen[identity] = true
entries = append(entries, entry)
}
for _, candidate := range runOutput.Candidates { for _, candidate := range runOutput.Candidates {
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] { if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
continue continue
@@ -197,6 +249,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []
continue continue
} }
out = append(out, LedgerValidatorDecisionRecord{ out = append(out, LedgerValidatorDecisionRecord{
ArtifactPath: decision.DiagnosticArtifactPath,
ValidatorKey: decision.ValidatorName, ValidatorKey: decision.ValidatorName,
Approved: decision.Approved, Approved: decision.Approved,
Disposition: decision.Disposition, Disposition: decision.Disposition,

View File

@@ -208,3 +208,37 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
} }
} }
} }
func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 3, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revised := original
revised.TargetSegmentID = 2
revised.OriginalText = "uh "
revised.CorrectedText = ""
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 1 {
t.Fatalf("ledger: %+v", ledger)
}
entry := ledger[0]
if entry.SegmentID != 1 || entry.OriginalText != "cat" || entry.ProposedCorrectedText == nil || *entry.ProposedCorrectedText != "dog" || entry.AppliedSegmentID == nil || *entry.AppliedSegmentID != 2 || entry.AppliedOriginalText == nil || *entry.AppliedOriginalText != "uh " || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "" || len(entry.RepairAttempts) != 1 || len(entry.DeterministicValidatorResults) != 1 || len(entry.RepairAttempts[0].LLMValidatorResults) != 1 || entry.RepairAttempts[0].LLMValidatorResults[0].ArtifactPath != "validation.json" {
t.Fatalf("lineage: %+v", entry)
}
}
func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revision := original
revision.CorrectedText = "fox"
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
var found *CorrectionLedgerEntry
for i := range ledger {
if ledger[i].Disposition == "interrupted" {
found = &ledger[i]
}
}
if found == nil || len(found.DeterministicValidatorResults) != 1 || len(found.RepairAttempts) != 1 || len(found.RepairAttempts[0].LLMValidatorResults) != 1 || found.AppliedCorrectedText != nil {
t.Fatalf("ledger: %+v", ledger)
}
}

View File

@@ -20,14 +20,15 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
} }
type executionCall struct { type executionCall struct {
CallID uint64 `json:"call_id"` RequestedPromptID string `json:"requested_prompt_id,omitempty"`
Context structuredoutput.DiagnosticContext `json:"context"` CallID uint64 `json:"call_id"`
Prompt structuredoutput.PromptMetadata `json:"prompt"` Context structuredoutput.DiagnosticContext `json:"context"`
Started time.Time `json:"started"` Prompt structuredoutput.PromptMetadata `json:"prompt"`
Finished *time.Time `json:"finished,omitempty"` Started time.Time `json:"started"`
ElapsedMS int64 `json:"elapsed_ms"` Finished *time.Time `json:"finished,omitempty"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"` ElapsedMS int64 `json:"elapsed_ms"`
Summary reporting.LLMSummary `json:"summary"` Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
} }
type executionEvent struct { type executionEvent struct {
@@ -71,7 +72,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
value := call.Finished value := call.Finished
finished = &value finished = &value
} }
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))}) calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
} }
for _, event := range snapshot.Events { for _, event := range snapshot.Events {
var generation, attempt *uint64 var generation, attempt *uint64

View File

@@ -3,6 +3,7 @@ package runner
import ( import (
"context" "context"
"fmt" "fmt"
"os"
"testing" "testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
t.Fatalf("findings: %+v", findings) t.Fatalf("findings: %+v", findings)
} }
} }
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
attempt := 1
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("decisions: %+v", result.decisions)
}
paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
}
if paths[0] == paths[1] {
t.Fatalf("colliding paths: %v", paths)
}
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatal(err)
}
}
}
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
t.Fatalf("result: %+v", result)
}
}

View File

@@ -4,7 +4,9 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"path/filepath"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -15,6 +17,7 @@ import (
// RepairAttempt retains compact facts for a single semantic request. // RepairAttempt retains compact facts for a single semantic request.
type RepairAttempt struct { type RepairAttempt struct {
Artifacts llm.InteractionArtifacts
Revision *proposals.EnrichedCorrectionProposal Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord Decisions []ValidatorDecisionRecord
Rejected []ValidatorRejectedChange Rejected []ValidatorRejectedChange
@@ -90,8 +93,38 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return initial, branches, err return initial, branches, err
} }
var response proposal_generation.RepairResponse var response proposal_generation.RepairResponse
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex}}, &response) proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
record := RepairAttempt{Counts: completion.Execution} record := RepairAttempt{Counts: completion.Execution}
save := func(err error) {
if input.DiagnosticsDir == "" {
return
}
var secrets []string
if source, ok := input.Spec.RepairClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets)
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure)
if writeErr != nil {
if input.ReportDiagnosticError != nil {
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
}
return
}
record.Artifacts = artifacts
}
if callErr != nil { if callErr != nil {
var failure *structuredoutput.Failure var failure *structuredoutput.Failure
if errors.As(callErr, &failure) { if errors.As(callErr, &failure) {
@@ -99,6 +132,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} else { } else {
record.FailureReason = outcomes.ReasonInternalFailure record.FailureReason = outcomes.ReasonInternalFailure
} }
save(callErr)
branch.Attempts = append(branch.Attempts, record) branch.Attempts = append(branch.Attempts, record)
if structuredoutput.IsRecoverable(callErr) { if structuredoutput.IsRecoverable(callErr) {
branch.Outcome = "unavailable" branch.Outcome = "unavailable"
@@ -110,6 +144,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1) decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil { if err != nil {
record.FailureReason = outcomes.ReasonInvalidOutput record.FailureReason = outcomes.ReasonInvalidOutput
save(err)
branch.Attempts = append(branch.Attempts, record) branch.Attempts = append(branch.Attempts, record)
branch.Outcome = "unavailable" branch.Outcome = "unavailable"
break break
@@ -118,6 +153,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
replay.Turns = append(replay.Turns, completion.RepairInstruction...) replay.Turns = append(replay.Turns, completion.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent}) replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
if decoded.Action == "withdraw" { if decoded.Action == "withdraw" {
save(nil)
branch.Attempts = append(branch.Attempts, record) branch.Attempts = append(branch.Attempts, record)
branch.Outcome = "withdrawn" branch.Outcome = "withdrawn"
break break
@@ -132,6 +168,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence} current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
branch.Final = current branch.Final = current
next := input next := input
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current} next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
validated, validationErr := validateSectionCandidates(ctx, next) validated, validationErr := validateSectionCandidates(ctx, next)
revisionSnapshot := current revisionSnapshot := current
@@ -144,9 +181,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
validated.operations[i].Reference.RepairAttempt = &attemptNumber validated.operations[i].Reference.RepairAttempt = &attemptNumber
} }
record.Operations = validated.operations record.Operations = validated.operations
save(validationErr)
branch.Attempts = append(branch.Attempts, record) branch.Attempts = append(branch.Attempts, record)
initial.operations = append(initial.operations, validated.operations...) initial.operations = append(initial.operations, validated.operations...)
initial.decisions = append(initial.decisions, validated.decisions...)
if validationErr != nil { if validationErr != nil {
branches = append(branches, branch) branches = append(branches, branch)
return initial, branches, validationErr return initial, branches, validationErr

View File

@@ -4,6 +4,9 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"os"
"path/filepath"
"strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,6 +23,7 @@ type repairClient struct {
outputs []string outputs []string
calls []structuredoutput.Request calls []structuredoutput.Request
err error err error
secrets []string
} }
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) { func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
if err := json.Unmarshal([]byte(value), out); err != nil { if err := json.Unmarshal([]byte(value), out); err != nil {
return structuredoutput.Response{}, err return structuredoutput.Response{}, err
} }
return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
} }
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) { func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}} client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
candidates := []proposals.EnrichedCorrectionProposal{ candidates := []proposals.EnrichedCorrectionProposal{
@@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
t.Fatalf("branches=%+v error=%v", branches, err) t.Fatalf("branches=%+v error=%v", branches, err)
} }
} }
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 2 {
t.Fatalf("branches: %+v", branches)
}
a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for _, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
}
}
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
warnings := 0
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
}
}

View File

@@ -646,7 +646,9 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
} }
type validateSectionCandidatesInput struct { type validateSectionCandidatesInput struct {
Exchange *structuredoutput.Exchange RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
// ReportDiagnosticError may be called concurrently; it must not affect processing. // ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"` ReportDiagnosticError func(error) `json:"-"`
Spec contracts.ModuleRunSpec Spec contracts.ModuleRunSpec
@@ -703,11 +705,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
} }
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript, WorkingTranscript: input.WorkingTranscript,
CandidateProposal: submitted, CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey, ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName, ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex, SectionIndex: input.SectionIndex,
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
ValidatorOrder: &validatorOrder, ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy, ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary, Glossary: input.Spec.Glossary,

View File

@@ -19,6 +19,8 @@ type DiagnosticContext struct {
SectionIndex *int `json:"section_index,omitempty"` SectionIndex *int `json:"section_index,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"` BatchIndex *int `json:"batch_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"` ValidatorName string `json:"validator_name,omitempty"`
ProposalIndex *int `json:"proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
} }
// Request names an embedded prompt and response schema. Profile selection and // Request names an embedded prompt and response schema. Profile selection and

View File

@@ -3,6 +3,7 @@ package validators
import ( import (
"context" "context"
"fmt" "fmt"
"math"
"strings" "strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
@@ -104,7 +105,7 @@ func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Reque
decisions = append(decisions, approval(c.ProposalIndex)) decisions = append(decisions, approval(c.ProposalIndex))
continue continue
} }
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy) preview := previewForEvidence(req, &segment, c.CorrectionProposal)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText { if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty")) decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue continue
@@ -154,7 +155,7 @@ func ValidateProtectedTerms(req Request) (Result, error) {
if segment.ID != c.TargetSegmentID { if segment.ID != c.TargetSegmentID {
continue continue
} }
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy) preview := previewForEvidence(req, segment, c.CorrectionProposal)
if preview.Applicable { if preview.Applicable {
if glossaryStage { if glossaryStage {
// A correction span may sit inside a longer exempt name. // A correction span may sit inside a longer exempt name.
@@ -177,3 +178,10 @@ func ValidateProtectedTerms(req Request) (Result, error) {
} }
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
} }
func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult {
if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy)
}

View File

@@ -11,7 +11,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
@@ -128,7 +127,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{ completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID, PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey, Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name}, Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
}, &response) }, &response)
batchIndex := batch.BatchIndex batchIndex := batch.BatchIndex
operation := outcomes.Operation{ operation := outcomes.Operation{
@@ -144,6 +143,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.SectionIndex != nil { if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex) stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
} }
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex) stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey) responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model} responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
@@ -155,12 +157,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
stage, stage,
map[string]any{ map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey, "stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name, "validator_name": v.name,
"validator_type": v.validatorType, "validator_type": v.validatorType,
"batch_index": batch.BatchIndex, "batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items), "proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt, "repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
"response_schema": responseSchema.DiagnosticsMap(), "prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
}, },
completion.Prepared, completion.Prepared,
responsePayload, responsePayload,
@@ -263,7 +266,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
} }
seg := segments[position] seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy) preview := previewForEvidence(req, &seg, p.CorrectionProposal)
if !preview.Applicable { if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation")) immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue continue

View File

@@ -36,6 +36,9 @@ type Request struct {
StageID *string `json:"stage_id,omitempty"` StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing. // ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"` ReportDiagnosticError func(error) `json:"-"`
IndependentEvidence bool `json:"-"`
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"` SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"` WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"` CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`