eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Retain repair diagnostics and correction lineage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:37:52 +00:00
parent 735dfc1834
commit f4c638313e
15 changed files with 308 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/framework/llm/observations.go
View File

@@ -53,13 +53,14 @@ type ExecutionObservation struct {
}
type LogicalCall struct {
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
WallDuration time.Duration `json:"wall_duration_ns"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
WallDuration time.Duration `json:"wall_duration_ns"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
}
type ObservationSnapshot struct {
@@ -92,6 +93,14 @@ func (c *ObservationCollector) Hooks() CallHooks {
}
func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext {
if value.ProposalIndex != nil {
index := *value.ProposalIndex
value.ProposalIndex = &index
}
if value.RepairAttempt != nil {
index := *value.RepairAttempt
value.RepairAttempt = &index
}
if value.StageIndex != nil {
index := *value.StageIndex
value.StageIndex = &index
@@ -117,7 +126,7 @@ func (c *ObservationCollector) recordCall(record structuredoutput.CallRecord) {
}
c.mu.Lock()
defer c.mu.Unlock()
call := LogicalCall{CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure}
call := LogicalCall{RequestedPromptID: record.RequestedPromptID, CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure}
if !record.Finished.IsZero() {
call.WallDuration = record.Finished.Sub(record.Started)
}

53
internal/framework/processreport/correction_ledger.go
View File

@@ -4,6 +4,7 @@ import (
"path/filepath"
"sort"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -33,6 +34,9 @@ type CorrectionLedgerEntry struct {
SegmentID int `json:"segment_id,omitempty"`
OriginalText string `json:"original_text,omitempty"`
ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"`
AppliedSegmentID *int `json:"applied_segment_id,omitempty"`
AppliedOriginalText *string `json:"applied_original_text,omitempty"`
RepairAttempts []LedgerRepairAttempt `json:"repair_attempts,omitempty"`
AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"`
ReplacementPolicy string `json:"replacement_policy,omitempty"`
Disposition string `json:"disposition"`
@@ -42,7 +46,20 @@ type CorrectionLedgerEntry struct {
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type LedgerRepairAttempt struct {
Attempt int `json:"attempt"`
Revision *proposals.EnrichedCorrectionProposal `json:"revision,omitempty"`
FailureReason string `json:"failure_reason,omitempty"`
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
RequestPayloadPath string `json:"request_payload_path,omitempty"`
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
}
type LedgerValidatorDecisionRecord struct {
ArtifactPath string `json:"artifact_path,omitempty"`
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
@@ -71,6 +88,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
policy string
decisions map[int][]runner.ValidatorDecisionRecord
}
repairs := make(map[proposalIdentity]runner.RepairOutcome)
for _, unit := range runOutput.WorkUnits {
for _, repair := range unit.Repairs {
repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair
}
}
seen := make(map[proposalIdentity]bool)
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
for _, module := range runOutput.ModuleResults {
@@ -85,6 +108,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
}
for _, change := range module.AppliedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
@@ -103,6 +129,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, change := range module.SkippedChanges {
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
continue
}
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
entries = append(entries, CorrectionLedgerEntry{
StageIndex: module.StageIndex, StageID: module.StageID,
@@ -122,6 +151,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
for _, rejection := range module.ValidatorRejected {
if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] {
continue
}
if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok {
continue
}
disposition := correctionDispositionRejected
if rejection.Disposition == validators.DispositionUnavailable {
disposition = correctionDispositionUnavailable
@@ -156,6 +191,23 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
})
}
}
for identity, repair := range repairs {
stage := stages[identity.instance]
original := repair.Original
entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)}
for i, attempt := range repair.Attempts {
item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)}
entry.RepairAttempts = append(entry.RepairAttempts, item)
}
if repair.Outcome == "applied" {
segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText
entry.AppliedSegmentID = &segment
entry.AppliedOriginalText = &before
entry.AppliedCorrectedText = &after
}
seen[identity] = true
entries = append(entries, entry)
}
for _, candidate := range runOutput.Candidates {
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
continue
@@ -197,6 +249,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []
continue
}
out = append(out, LedgerValidatorDecisionRecord{
ArtifactPath: decision.DiagnosticArtifactPath,
ValidatorKey: decision.ValidatorName,
Approved: decision.Approved,
Disposition: decision.Disposition,

34
internal/framework/processreport/correction_ledger_test.go
View File

@@ -208,3 +208,37 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
}
}
}
func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 3, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revised := original
revised.TargetSegmentID = 2
revised.OriginalText = "uh "
revised.CorrectedText = ""
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 1 {
t.Fatalf("ledger: %+v", ledger)
}
entry := ledger[0]
if entry.SegmentID != 1 || entry.OriginalText != "cat" || entry.ProposedCorrectedText == nil || *entry.ProposedCorrectedText != "dog" || entry.AppliedSegmentID == nil || *entry.AppliedSegmentID != 2 || entry.AppliedOriginalText == nil || *entry.AppliedOriginalText != "uh " || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "" || len(entry.RepairAttempts) != 1 || len(entry.DeterministicValidatorResults) != 1 || len(entry.RepairAttempts[0].LLMValidatorResults) != 1 || entry.RepairAttempts[0].LLMValidatorResults[0].ArtifactPath != "validation.json" {
t.Fatalf("lineage: %+v", entry)
}
}
func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revision := original
revision.CorrectedText = "fox"
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
var found *CorrectionLedgerEntry
for i := range ledger {
if ledger[i].Disposition == "interrupted" {
found = &ledger[i]
}
}
if found == nil || len(found.DeterministicValidatorResults) != 1 || len(found.RepairAttempts) != 1 || len(found.RepairAttempts[0].LLMValidatorResults) != 1 || found.AppliedCorrectedText != nil {
t.Fatalf("ledger: %+v", ledger)
}
}

19
internal/framework/processreport/llm.go
View File

@@ -20,14 +20,15 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
}
type executionCall struct {
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
}
type executionEvent struct {
@@ -71,7 +72,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
value := call.Finished
finished = &value
}
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
}
for _, event := range snapshot.Events {
var generation, attempt *uint64

52
internal/framework/runner/feedback_test.go
View File

@@ -3,6 +3,7 @@ package runner
import (
"context"
"fmt"
"os"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
t.Fatalf("findings: %+v", findings)
}
}
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
attempt := 1
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
t.Fatalf("decisions: %+v", result.decisions)
}
paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
}
if paths[0] == paths[1] {
t.Fatalf("colliding paths: %v", paths)
}
for _, path := range paths {
if _, err := os.Stat(path); err != nil {
t.Fatal(err)
}
}
}
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
t.Fatalf("result: %+v", result)
}
}

41
internal/framework/runner/repair.go
View File

@@ -4,7 +4,9 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -15,6 +17,7 @@ import (
// RepairAttempt retains compact facts for a single semantic request.
type RepairAttempt struct {
Artifacts llm.InteractionArtifacts
Revision *proposals.EnrichedCorrectionProposal
Decisions []ValidatorDecisionRecord
Rejected []ValidatorRejectedChange
@@ -90,8 +93,38 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return initial, branches, err
}
var response proposal_generation.RepairResponse
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex}}, &response)
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
record := RepairAttempt{Counts: completion.Execution}
save := func(err error) {
if input.DiagnosticsDir == "" {
return
}
var secrets []string
if source, ok := input.Spec.RepairClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
section := 0
if input.SectionIndex != nil {
section = *input.SectionIndex
}
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
var failure any
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets)
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure)
if writeErr != nil {
if input.ReportDiagnosticError != nil {
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
}
return
}
record.Artifacts = artifacts
}
if callErr != nil {
var failure *structuredoutput.Failure
if errors.As(callErr, &failure) {
@@ -99,6 +132,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} else {
record.FailureReason = outcomes.ReasonInternalFailure
}
save(callErr)
branch.Attempts = append(branch.Attempts, record)
if structuredoutput.IsRecoverable(callErr) {
branch.Outcome = "unavailable"
@@ -110,6 +144,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil {
record.FailureReason = outcomes.ReasonInvalidOutput
save(err)
branch.Attempts = append(branch.Attempts, record)
branch.Outcome = "unavailable"
break
@@ -118,6 +153,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
if decoded.Action == "withdraw" {
save(nil)
branch.Attempts = append(branch.Attempts, record)
branch.Outcome = "withdrawn"
break
@@ -132,6 +168,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
branch.Final = current
next := input
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
validated, validationErr := validateSectionCandidates(ctx, next)
revisionSnapshot := current
@@ -144,9 +181,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
record.Operations = validated.operations
save(validationErr)
branch.Attempts = append(branch.Attempts, record)
initial.operations = append(initial.operations, validated.operations...)
initial.decisions = append(initial.decisions, validated.decisions...)
if validationErr != nil {
branches = append(branches, branch)
return initial, branches, validationErr

60
internal/framework/runner/repair_test.go
View File

@@ -4,6 +4,9 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,6 +23,7 @@ type repairClient struct {
outputs []string
calls []structuredoutput.Request
err error
secrets []string
}
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
if err := json.Unmarshal([]byte(value), out); err != nil {
return structuredoutput.Response{}, err
}
return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
}
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
@@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
t.Fatalf("branches=%+v error=%v", branches, err)
}
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 2 {
t.Fatalf("branches: %+v", branches)
}
a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for _, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
}
}
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
warnings := 0
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
}
}

15
internal/framework/runner/runner.go
View File

@@ -646,7 +646,9 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Exchange *structuredoutput.Exchange
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
Spec contracts.ModuleRunSpec
@@ -703,11 +705,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
WorkingTranscript: input.WorkingTranscript,
CandidateProposal: submitted,
ModuleKey: input.Spec.ModuleKey,
ModuleInstance: input.Spec.InstanceName,
SectionIndex: input.SectionIndex,
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
ValidatorOrder: &validatorOrder,
ReplacementPolicy: input.Policy,
Glossary: input.Spec.Glossary,

2
internal/framework/structuredoutput/call.go
View File

@@ -19,6 +19,8 @@ type DiagnosticContext struct {
SectionIndex *int `json:"section_index,omitempty"`
BatchIndex *int `json:"batch_index,omitempty"`
ValidatorName string `json:"validator_name,omitempty"`
ProposalIndex *int `json:"proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
}
// Request names an embedded prompt and response schema. Profile selection and

12
internal/framework/validators/deterministic.go
View File

@@ -3,6 +3,7 @@ package validators
import (
"context"
"fmt"
"math"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
@@ -104,7 +105,7 @@ func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Reque
decisions = append(decisions, approval(c.ProposalIndex))
continue
}
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, &segment, c.CorrectionProposal)
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
continue
@@ -154,7 +155,7 @@ func ValidateProtectedTerms(req Request) (Result, error) {
if segment.ID != c.TargetSegmentID {
continue
}
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, segment, c.CorrectionProposal)
if preview.Applicable {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
@@ -177,3 +178,10 @@ func ValidateProtectedTerms(req Request) (Result, error) {
}
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
}
func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult {
if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy)
}

21
internal/framework/validators/llm_validators.go
View File

@@ -11,7 +11,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
@@ -128,7 +127,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
PromptID: promptMetadata.PromptID,
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
}, &response)
batchIndex := batch.BatchIndex
operation := outcomes.Operation{
@@ -144,6 +143,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.SectionIndex != nil {
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
}
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
}
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
@@ -155,12 +157,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
stage,
map[string]any{
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
"validator_name": v.name,
"validator_type": v.validatorType,
"batch_index": batch.BatchIndex,
"proposal_indices": batchProposalIndices(batch.Items),
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
"prompt": completion.Prompt,
"response_schema": responseSchema.DiagnosticsMap(),
},
completion.Prepared,
responsePayload,
@@ -263,7 +266,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
}
seg := segments[position]
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
preview := previewForEvidence(req, &seg, p.CorrectionProposal)
if !preview.Applicable {
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
continue

3
internal/framework/validators/models.go
View File

@@ -36,6 +36,9 @@ type Request struct {
StageID *string `json:"stage_id,omitempty"`
// ReportDiagnosticError may be called concurrently; it must not affect processing.
ReportDiagnosticError func(error) `json:"-"`
IndependentEvidence bool `json:"-"`
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
RepairAttempt *int `json:"repair_attempt,omitempty"`
SectionIndex *int `json:"section_index,omitempty"`
WorkingTranscript *schema.Transcript `json:"-"`
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`