Retain repair diagnostics and correction lineage
This commit is contained in:
@@ -101,6 +101,9 @@ holds logical calls and PromptKit execution events. Interaction request/response
|
|||||||
artifacts and input snapshots are also debug-only. Records carry zero-based
|
artifacts and input snapshots are also debug-only. Records carry zero-based
|
||||||
`stage_index` and optional `stage_id` where an entry is identified.
|
`stage_index` and optional `stage_id` where an entry is identified.
|
||||||
These files are diagnostic evidence, not an alternate ordinary report schema.
|
These files are diagnostic evidence, not an alternate ordinary report schema.
|
||||||
|
For a repaired proposal, one ledger row preserves the original proposed edit,
|
||||||
|
the final applied segment and span when applicable, and repair attempts with
|
||||||
|
revision decisions and interaction artifact paths.
|
||||||
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
||||||
literal replacement span, even when it is `""` for a deletion. Applied entries
|
literal replacement span, even when it is `""` for a deletion. Applied entries
|
||||||
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
||||||
|
|||||||
@@ -37,7 +37,9 @@ under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
|
|||||||
when debug is off. With debug enabled, it captures source and normalized
|
when debug is off. With debug enabled, it captures source and normalized
|
||||||
snapshots, invocation, effective settings, correction ledger, LLM execution
|
snapshots, invocation, effective settings, correction ledger, LLM execution
|
||||||
observations, and model interactions. Candidate capture includes proposals later
|
observations, and model interactions. Candidate capture includes proposals later
|
||||||
rejected or interrupted. Ledger mapping preserves empty proposed and applied
|
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
|
||||||
|
attempt, validator, and batch coordinates as applicable; their retained content
|
||||||
|
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
|
||||||
replacements and distinguishes unavailable candidates from interrupted work.
|
replacements and distinguishes unavailable candidates from interrupted work.
|
||||||
|
|
||||||
A concurrency-safe diagnostic callback reports optional write failures without
|
A concurrency-safe diagnostic callback reports optional write failures without
|
||||||
|
|||||||
@@ -322,6 +322,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
|
|||||||
|
|
||||||
## Stage 8 — Retain optional repair evidence and preserve ledger lineage
|
## Stage 8 — Retain optional repair evidence and preserve ledger lineage
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
1. Extend the current interaction writers and logical execution observation
|
1. Extend the current interaction writers and logical execution observation
|
||||||
data with a repair role, original proposal index, and one-based semantic
|
data with a repair role, original proposal index, and one-based semantic
|
||||||
attempt number. Use generated stage instance, section, proposal, and
|
attempt number. Use generated stage instance, section, proposal, and
|
||||||
|
|||||||
@@ -53,13 +53,14 @@ type ExecutionObservation struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type LogicalCall struct {
|
type LogicalCall struct {
|
||||||
CallID uint64 `json:"call_id"`
|
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
CallID uint64 `json:"call_id"`
|
||||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||||
Started time.Time `json:"started"`
|
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||||
Finished time.Time `json:"finished,omitempty"`
|
Started time.Time `json:"started"`
|
||||||
WallDuration time.Duration `json:"wall_duration_ns"`
|
Finished time.Time `json:"finished,omitempty"`
|
||||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
WallDuration time.Duration `json:"wall_duration_ns"`
|
||||||
|
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ObservationSnapshot struct {
|
type ObservationSnapshot struct {
|
||||||
@@ -92,6 +93,14 @@ func (c *ObservationCollector) Hooks() CallHooks {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext {
|
func copyContext(value structuredoutput.DiagnosticContext) structuredoutput.DiagnosticContext {
|
||||||
|
if value.ProposalIndex != nil {
|
||||||
|
index := *value.ProposalIndex
|
||||||
|
value.ProposalIndex = &index
|
||||||
|
}
|
||||||
|
if value.RepairAttempt != nil {
|
||||||
|
index := *value.RepairAttempt
|
||||||
|
value.RepairAttempt = &index
|
||||||
|
}
|
||||||
if value.StageIndex != nil {
|
if value.StageIndex != nil {
|
||||||
index := *value.StageIndex
|
index := *value.StageIndex
|
||||||
value.StageIndex = &index
|
value.StageIndex = &index
|
||||||
@@ -117,7 +126,7 @@ func (c *ObservationCollector) recordCall(record structuredoutput.CallRecord) {
|
|||||||
}
|
}
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
call := LogicalCall{CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure}
|
call := LogicalCall{RequestedPromptID: record.RequestedPromptID, CallID: record.CallID, Context: copyContext(record.Context), Prompt: record.Prompt, Started: record.Started, Finished: record.Finished, Failure: record.Failure}
|
||||||
if !record.Finished.IsZero() {
|
if !record.Finished.IsZero() {
|
||||||
call.WallDuration = record.Finished.Sub(record.Started)
|
call.WallDuration = record.Finished.Sub(record.Started)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||||
@@ -33,6 +34,9 @@ type CorrectionLedgerEntry struct {
|
|||||||
SegmentID int `json:"segment_id,omitempty"`
|
SegmentID int `json:"segment_id,omitempty"`
|
||||||
OriginalText string `json:"original_text,omitempty"`
|
OriginalText string `json:"original_text,omitempty"`
|
||||||
ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"`
|
ProposedCorrectedText *string `json:"proposed_corrected_text,omitempty"`
|
||||||
|
AppliedSegmentID *int `json:"applied_segment_id,omitempty"`
|
||||||
|
AppliedOriginalText *string `json:"applied_original_text,omitempty"`
|
||||||
|
RepairAttempts []LedgerRepairAttempt `json:"repair_attempts,omitempty"`
|
||||||
AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"`
|
AppliedCorrectedText *string `json:"applied_corrected_text,omitempty"`
|
||||||
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
ReplacementPolicy string `json:"replacement_policy,omitempty"`
|
||||||
Disposition string `json:"disposition"`
|
Disposition string `json:"disposition"`
|
||||||
@@ -42,7 +46,20 @@ type CorrectionLedgerEntry struct {
|
|||||||
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type LedgerRepairAttempt struct {
|
||||||
|
Attempt int `json:"attempt"`
|
||||||
|
Revision *proposals.EnrichedCorrectionProposal `json:"revision,omitempty"`
|
||||||
|
FailureReason string `json:"failure_reason,omitempty"`
|
||||||
|
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||||
|
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||||
|
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||||
|
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||||
|
DeterministicValidatorResults []LedgerValidatorDecisionRecord `json:"deterministic_validator_decisions,omitempty"`
|
||||||
|
LLMValidatorResults []LedgerValidatorDecisionRecord `json:"llm_validator_decisions,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type LedgerValidatorDecisionRecord struct {
|
type LedgerValidatorDecisionRecord struct {
|
||||||
|
ArtifactPath string `json:"artifact_path,omitempty"`
|
||||||
ValidatorKey string `json:"validator_key"`
|
ValidatorKey string `json:"validator_key"`
|
||||||
Approved bool `json:"approved"`
|
Approved bool `json:"approved"`
|
||||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||||
@@ -71,6 +88,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
|||||||
policy string
|
policy string
|
||||||
decisions map[int][]runner.ValidatorDecisionRecord
|
decisions map[int][]runner.ValidatorDecisionRecord
|
||||||
}
|
}
|
||||||
|
repairs := make(map[proposalIdentity]runner.RepairOutcome)
|
||||||
|
for _, unit := range runOutput.WorkUnits {
|
||||||
|
for _, repair := range unit.Repairs {
|
||||||
|
repairs[proposalIdentity{unit.ModuleInstance, repair.ProposalIndex}] = repair
|
||||||
|
}
|
||||||
|
}
|
||||||
seen := make(map[proposalIdentity]bool)
|
seen := make(map[proposalIdentity]bool)
|
||||||
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
|
stages := make(map[string]stageEvidence, len(runOutput.ModuleResults))
|
||||||
for _, module := range runOutput.ModuleResults {
|
for _, module := range runOutput.ModuleResults {
|
||||||
@@ -85,6 +108,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, change := range module.AppliedChanges {
|
for _, change := range module.AppliedChanges {
|
||||||
|
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
|
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
|
||||||
entries = append(entries, CorrectionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||||
@@ -103,6 +129,9 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, change := range module.SkippedChanges {
|
for _, change := range module.SkippedChanges {
|
||||||
|
if _, ok := repairs[proposalIdentity{module.ModuleInstance, change.ProposalIndex}]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
|
seen[proposalIdentity{module.ModuleInstance, change.ProposalIndex}] = true
|
||||||
entries = append(entries, CorrectionLedgerEntry{
|
entries = append(entries, CorrectionLedgerEntry{
|
||||||
StageIndex: module.StageIndex, StageID: module.StageID,
|
StageIndex: module.StageIndex, StageID: module.StageID,
|
||||||
@@ -122,6 +151,12 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, rejection := range module.ValidatorRejected {
|
for _, rejection := range module.ValidatorRejected {
|
||||||
|
if seen[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := repairs[proposalIdentity{module.ModuleInstance, rejection.ProposalIndex}]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
disposition := correctionDispositionRejected
|
disposition := correctionDispositionRejected
|
||||||
if rejection.Disposition == validators.DispositionUnavailable {
|
if rejection.Disposition == validators.DispositionUnavailable {
|
||||||
disposition = correctionDispositionUnavailable
|
disposition = correctionDispositionUnavailable
|
||||||
@@ -156,6 +191,23 @@ func BuildCorrectionLedger(input CorrectionLedgerInput) []CorrectionLedgerEntry
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for identity, repair := range repairs {
|
||||||
|
stage := stages[identity.instance]
|
||||||
|
original := repair.Original
|
||||||
|
entry := CorrectionLedgerEntry{StageIndex: stage.index, StageID: stage.id, RunID: runID, ModuleKey: original.ModuleKey, ModuleInstance: identity.instance, ProposalIndex: identity.index, SegmentID: original.TargetSegmentID, OriginalText: original.OriginalText, ProposedCorrectedText: &original.CorrectedText, ReplacementPolicy: stage.policy, Disposition: repair.Outcome, DeterministicValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], false), LLMValidatorResults: filterLedgerDecisions(stage.decisions[identity.index], true)}
|
||||||
|
for i, attempt := range repair.Attempts {
|
||||||
|
item := LedgerRepairAttempt{Attempt: i + 1, Revision: attempt.Revision, FailureReason: string(attempt.FailureReason), RequestMetadataPath: attempt.Artifacts.RequestMetadataPath, RequestPayloadPath: attempt.Artifacts.RequestPayloadPath, ResponsePayloadPath: attempt.Artifacts.ResponsePayloadPath, ErrorPayloadPath: attempt.Artifacts.ErrorPayloadPath, DeterministicValidatorResults: filterLedgerDecisions(attempt.Decisions, false), LLMValidatorResults: filterLedgerDecisions(attempt.Decisions, true)}
|
||||||
|
entry.RepairAttempts = append(entry.RepairAttempts, item)
|
||||||
|
}
|
||||||
|
if repair.Outcome == "applied" {
|
||||||
|
segment, before, after := repair.Final.TargetSegmentID, repair.Final.OriginalText, repair.Final.CorrectedText
|
||||||
|
entry.AppliedSegmentID = &segment
|
||||||
|
entry.AppliedOriginalText = &before
|
||||||
|
entry.AppliedCorrectedText = &after
|
||||||
|
}
|
||||||
|
seen[identity] = true
|
||||||
|
entries = append(entries, entry)
|
||||||
|
}
|
||||||
for _, candidate := range runOutput.Candidates {
|
for _, candidate := range runOutput.Candidates {
|
||||||
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
|
if seen[proposalIdentity{candidate.ModuleInstance, candidate.ProposalIndex}] {
|
||||||
continue
|
continue
|
||||||
@@ -197,6 +249,7 @@ func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, LedgerValidatorDecisionRecord{
|
out = append(out, LedgerValidatorDecisionRecord{
|
||||||
|
ArtifactPath: decision.DiagnosticArtifactPath,
|
||||||
ValidatorKey: decision.ValidatorName,
|
ValidatorKey: decision.ValidatorName,
|
||||||
Approved: decision.Approved,
|
Approved: decision.Approved,
|
||||||
Disposition: decision.Disposition,
|
Disposition: decision.Disposition,
|
||||||
|
|||||||
@@ -208,3 +208,37 @@ func TestCorrectionLedgerSerializesEmptyReplacementByDisposition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
|
||||||
|
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 3, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
|
||||||
|
revised := original
|
||||||
|
revised.TargetSegmentID = 2
|
||||||
|
revised.OriginalText = "uh "
|
||||||
|
revised.CorrectedText = ""
|
||||||
|
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||||
|
if len(ledger) != 1 {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
entry := ledger[0]
|
||||||
|
if entry.SegmentID != 1 || entry.OriginalText != "cat" || entry.ProposedCorrectedText == nil || *entry.ProposedCorrectedText != "dog" || entry.AppliedSegmentID == nil || *entry.AppliedSegmentID != 2 || entry.AppliedOriginalText == nil || *entry.AppliedOriginalText != "uh " || entry.AppliedCorrectedText == nil || *entry.AppliedCorrectedText != "" || len(entry.RepairAttempts) != 1 || len(entry.DeterministicValidatorResults) != 1 || len(entry.RepairAttempts[0].LLMValidatorResults) != 1 || entry.RepairAttempts[0].LLMValidatorResults[0].ArtifactPath != "validation.json" {
|
||||||
|
t.Fatalf("lineage: %+v", entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
|
||||||
|
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
|
||||||
|
revision := original
|
||||||
|
revision.CorrectedText = "fox"
|
||||||
|
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||||
|
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||||
|
var found *CorrectionLedgerEntry
|
||||||
|
for i := range ledger {
|
||||||
|
if ledger[i].Disposition == "interrupted" {
|
||||||
|
found = &ledger[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if found == nil || len(found.DeterministicValidatorResults) != 1 || len(found.RepairAttempts) != 1 || len(found.RepairAttempts[0].LLMValidatorResults) != 1 || found.AppliedCorrectedText != nil {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,14 +20,15 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type executionCall struct {
|
type executionCall struct {
|
||||||
CallID uint64 `json:"call_id"`
|
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
CallID uint64 `json:"call_id"`
|
||||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||||
Started time.Time `json:"started"`
|
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||||
Finished *time.Time `json:"finished,omitempty"`
|
Started time.Time `json:"started"`
|
||||||
ElapsedMS int64 `json:"elapsed_ms"`
|
Finished *time.Time `json:"finished,omitempty"`
|
||||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
ElapsedMS int64 `json:"elapsed_ms"`
|
||||||
Summary reporting.LLMSummary `json:"summary"`
|
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||||
|
Summary reporting.LLMSummary `json:"summary"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type executionEvent struct {
|
type executionEvent struct {
|
||||||
@@ -71,7 +72,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
|||||||
value := call.Finished
|
value := call.Finished
|
||||||
finished = &value
|
finished = &value
|
||||||
}
|
}
|
||||||
calls = append(calls, executionCall{CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||||
}
|
}
|
||||||
for _, event := range snapshot.Events {
|
for _, event := range snapshot.Events {
|
||||||
var generation, attempt *uint64
|
var generation, attempt *uint64
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package runner
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
@@ -85,3 +86,54 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
|||||||
t.Fatalf("findings: %+v", findings)
|
t.Fatalf("findings: %+v", findings)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||||
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||||
|
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||||
|
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||||
|
}}
|
||||||
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
attempt := 1
|
||||||
|
var paths []string
|
||||||
|
for _, index := range []int{2, 3} {
|
||||||
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||||
|
result, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.decisions) != 1 || result.decisions[0].DiagnosticArtifactPath == "" {
|
||||||
|
t.Fatalf("decisions: %+v", result.decisions)
|
||||||
|
}
|
||||||
|
paths = append(paths, result.decisions[0].DiagnosticArtifactPath)
|
||||||
|
}
|
||||||
|
if paths[0] == paths[1] {
|
||||||
|
t.Fatalf("colliding paths: %v", paths)
|
||||||
|
}
|
||||||
|
for _, path := range paths {
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
|
||||||
|
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
||||||
|
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
|
||||||
|
result, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
|
||||||
|
t.Fatalf("result: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
@@ -15,6 +17,7 @@ import (
|
|||||||
|
|
||||||
// RepairAttempt retains compact facts for a single semantic request.
|
// RepairAttempt retains compact facts for a single semantic request.
|
||||||
type RepairAttempt struct {
|
type RepairAttempt struct {
|
||||||
|
Artifacts llm.InteractionArtifacts
|
||||||
Revision *proposals.EnrichedCorrectionProposal
|
Revision *proposals.EnrichedCorrectionProposal
|
||||||
Decisions []ValidatorDecisionRecord
|
Decisions []ValidatorDecisionRecord
|
||||||
Rejected []ValidatorRejectedChange
|
Rejected []ValidatorRejectedChange
|
||||||
@@ -90,8 +93,38 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
return initial, branches, err
|
return initial, branches, err
|
||||||
}
|
}
|
||||||
var response proposal_generation.RepairResponse
|
var response proposal_generation.RepairResponse
|
||||||
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex}}, &response)
|
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||||
|
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
|
||||||
record := RepairAttempt{Counts: completion.Execution}
|
record := RepairAttempt{Counts: completion.Execution}
|
||||||
|
save := func(err error) {
|
||||||
|
if input.DiagnosticsDir == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var secrets []string
|
||||||
|
if source, ok := input.Spec.RepairClient.(interface {
|
||||||
|
ConfiguredSecrets(context.Context) []string
|
||||||
|
}); ok {
|
||||||
|
secrets = source.ConfiguredSecrets(ctx)
|
||||||
|
}
|
||||||
|
section := 0
|
||||||
|
if input.SectionIndex != nil {
|
||||||
|
section = *input.SectionIndex
|
||||||
|
}
|
||||||
|
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, proposalIndex, attemptNumber)
|
||||||
|
var failure any
|
||||||
|
if err != nil {
|
||||||
|
failure = map[string]string{"error": err.Error()}
|
||||||
|
}
|
||||||
|
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), secrets)
|
||||||
|
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": record.Rejected}, failure)
|
||||||
|
if writeErr != nil {
|
||||||
|
if input.ReportDiagnosticError != nil {
|
||||||
|
input.ReportDiagnosticError(fmt.Errorf("repair interaction %s: %w", name, writeErr))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
record.Artifacts = artifacts
|
||||||
|
}
|
||||||
if callErr != nil {
|
if callErr != nil {
|
||||||
var failure *structuredoutput.Failure
|
var failure *structuredoutput.Failure
|
||||||
if errors.As(callErr, &failure) {
|
if errors.As(callErr, &failure) {
|
||||||
@@ -99,6 +132,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
} else {
|
} else {
|
||||||
record.FailureReason = outcomes.ReasonInternalFailure
|
record.FailureReason = outcomes.ReasonInternalFailure
|
||||||
}
|
}
|
||||||
|
save(callErr)
|
||||||
branch.Attempts = append(branch.Attempts, record)
|
branch.Attempts = append(branch.Attempts, record)
|
||||||
if structuredoutput.IsRecoverable(callErr) {
|
if structuredoutput.IsRecoverable(callErr) {
|
||||||
branch.Outcome = "unavailable"
|
branch.Outcome = "unavailable"
|
||||||
@@ -110,6 +144,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
record.FailureReason = outcomes.ReasonInvalidOutput
|
record.FailureReason = outcomes.ReasonInvalidOutput
|
||||||
|
save(err)
|
||||||
branch.Attempts = append(branch.Attempts, record)
|
branch.Attempts = append(branch.Attempts, record)
|
||||||
branch.Outcome = "unavailable"
|
branch.Outcome = "unavailable"
|
||||||
break
|
break
|
||||||
@@ -118,6 +153,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
|
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
|
||||||
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
|
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
|
||||||
if decoded.Action == "withdraw" {
|
if decoded.Action == "withdraw" {
|
||||||
|
save(nil)
|
||||||
branch.Attempts = append(branch.Attempts, record)
|
branch.Attempts = append(branch.Attempts, record)
|
||||||
branch.Outcome = "withdrawn"
|
branch.Outcome = "withdrawn"
|
||||||
break
|
break
|
||||||
@@ -132,6 +168,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
|
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
|
||||||
branch.Final = current
|
branch.Final = current
|
||||||
next := input
|
next := input
|
||||||
|
next.RepairProposalIndex, next.RepairAttempt = &proposalIndex, &attemptNumber
|
||||||
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
|
next.SectionEnriched = []proposals.EnrichedCorrectionProposal{current}
|
||||||
validated, validationErr := validateSectionCandidates(ctx, next)
|
validated, validationErr := validateSectionCandidates(ctx, next)
|
||||||
revisionSnapshot := current
|
revisionSnapshot := current
|
||||||
@@ -144,9 +181,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
|||||||
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
||||||
}
|
}
|
||||||
record.Operations = validated.operations
|
record.Operations = validated.operations
|
||||||
|
save(validationErr)
|
||||||
branch.Attempts = append(branch.Attempts, record)
|
branch.Attempts = append(branch.Attempts, record)
|
||||||
initial.operations = append(initial.operations, validated.operations...)
|
initial.operations = append(initial.operations, validated.operations...)
|
||||||
initial.decisions = append(initial.decisions, validated.decisions...)
|
|
||||||
if validationErr != nil {
|
if validationErr != nil {
|
||||||
branches = append(branches, branch)
|
branches = append(branches, branch)
|
||||||
return initial, branches, validationErr
|
return initial, branches, validationErr
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
@@ -20,6 +23,7 @@ type repairClient struct {
|
|||||||
outputs []string
|
outputs []string
|
||||||
calls []structuredoutput.Request
|
calls []structuredoutput.Request
|
||||||
err error
|
err error
|
||||||
|
secrets []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||||
@@ -36,9 +40,11 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
|||||||
if err := json.Unmarshal([]byte(value), out); err != nil {
|
if err := json.Unmarshal([]byte(value), out); err != nil {
|
||||||
return structuredoutput.Response{}, err
|
return structuredoutput.Response{}, err
|
||||||
}
|
}
|
||||||
return structuredoutput.Response{Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
|
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||||
|
|
||||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
|
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
|
||||||
candidates := []proposals.EnrichedCorrectionProposal{
|
candidates := []proposals.EnrichedCorrectionProposal{
|
||||||
@@ -182,3 +188,55 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
|||||||
t.Fatalf("branches=%+v error=%v", branches, err)
|
t.Fatalf("branches=%+v error=%v", branches, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||||
|
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||||
|
candidates := []proposals.EnrichedCorrectionProposal{
|
||||||
|
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||||
|
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||||
|
}
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||||
|
initial, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(branches) != 2 {
|
||||||
|
t.Fatalf("branches: %+v", branches)
|
||||||
|
}
|
||||||
|
a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
|
||||||
|
if a == "" || a == b {
|
||||||
|
t.Fatalf("artifact paths: %q %q", a, b)
|
||||||
|
}
|
||||||
|
for _, path := range []string{a, b} {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
|
||||||
|
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||||
|
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||||
|
path := filepath.Join(t.TempDir(), "file")
|
||||||
|
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
warnings := 0
|
||||||
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||||
|
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||||
|
initial, err := validateSectionCandidates(context.Background(), input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||||
|
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || warnings != 1 {
|
||||||
|
t.Fatalf("branches=%+v warnings=%d err=%v", branches, warnings, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -646,7 +646,9 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
}
|
}
|
||||||
|
|
||||||
type validateSectionCandidatesInput struct {
|
type validateSectionCandidatesInput struct {
|
||||||
Exchange *structuredoutput.Exchange
|
RepairProposalIndex *int
|
||||||
|
RepairAttempt *int
|
||||||
|
Exchange *structuredoutput.Exchange
|
||||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
Spec contracts.ModuleRunSpec
|
Spec contracts.ModuleRunSpec
|
||||||
@@ -703,11 +705,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
}
|
}
|
||||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||||
WorkingTranscript: input.WorkingTranscript,
|
WorkingTranscript: input.WorkingTranscript,
|
||||||
CandidateProposal: submitted,
|
CandidateProposal: submitted,
|
||||||
ModuleKey: input.Spec.ModuleKey,
|
ModuleKey: input.Spec.ModuleKey,
|
||||||
ModuleInstance: input.Spec.InstanceName,
|
ModuleInstance: input.Spec.InstanceName,
|
||||||
SectionIndex: input.SectionIndex,
|
SectionIndex: input.SectionIndex,
|
||||||
|
IndependentEvidence: feedback, RepairProposalIndex: input.RepairProposalIndex, RepairAttempt: input.RepairAttempt,
|
||||||
ValidatorOrder: &validatorOrder,
|
ValidatorOrder: &validatorOrder,
|
||||||
ReplacementPolicy: input.Policy,
|
ReplacementPolicy: input.Policy,
|
||||||
Glossary: input.Spec.Glossary,
|
Glossary: input.Spec.Glossary,
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ type DiagnosticContext struct {
|
|||||||
SectionIndex *int `json:"section_index,omitempty"`
|
SectionIndex *int `json:"section_index,omitempty"`
|
||||||
BatchIndex *int `json:"batch_index,omitempty"`
|
BatchIndex *int `json:"batch_index,omitempty"`
|
||||||
ValidatorName string `json:"validator_name,omitempty"`
|
ValidatorName string `json:"validator_name,omitempty"`
|
||||||
|
ProposalIndex *int `json:"proposal_index,omitempty"`
|
||||||
|
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request names an embedded prompt and response schema. Profile selection and
|
// Request names an embedded prompt and response schema. Profile selection and
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package validators
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
@@ -104,7 +105,7 @@ func (v NonEmptyResultingSegmentValidator) Validate(_ context.Context, req Reque
|
|||||||
decisions = append(decisions, approval(c.ProposalIndex))
|
decisions = append(decisions, approval(c.ProposalIndex))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
preview := proposals.PreviewProposalForSegment(&segment, c.CorrectionProposal, req.ReplacementPolicy)
|
preview := previewForEvidence(req, &segment, c.CorrectionProposal)
|
||||||
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
|
if preview.SkipReason == proposals.SkipReasonEmptyResultingText {
|
||||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
|
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyResultingText, "proposal would leave the segment empty"))
|
||||||
continue
|
continue
|
||||||
@@ -154,7 +155,7 @@ func ValidateProtectedTerms(req Request) (Result, error) {
|
|||||||
if segment.ID != c.TargetSegmentID {
|
if segment.ID != c.TargetSegmentID {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy)
|
preview := previewForEvidence(req, segment, c.CorrectionProposal)
|
||||||
if preview.Applicable {
|
if preview.Applicable {
|
||||||
if glossaryStage {
|
if glossaryStage {
|
||||||
// A correction span may sit inside a longer exempt name.
|
// A correction span may sit inside a longer exempt name.
|
||||||
@@ -177,3 +178,10 @@ func ValidateProtectedTerms(req Request) (Result, error) {
|
|||||||
}
|
}
|
||||||
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
|
return Result{ValidatorName: "protected_terms", Decisions: decisions}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func previewForEvidence(req Request, segment *schema.Segment, correction proposals.CorrectionProposal) proposals.SegmentPreviewResult {
|
||||||
|
if req.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
|
||||||
|
correction.Confidence = 0.5
|
||||||
|
}
|
||||||
|
return proposals.PreviewProposalForSegment(segment, correction, req.ReplacementPolicy)
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
@@ -128,7 +127,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
||||||
PromptID: promptMetadata.PromptID,
|
PromptID: promptMetadata.PromptID,
|
||||||
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
||||||
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name},
|
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
|
||||||
}, &response)
|
}, &response)
|
||||||
batchIndex := batch.BatchIndex
|
batchIndex := batch.BatchIndex
|
||||||
operation := outcomes.Operation{
|
operation := outcomes.Operation{
|
||||||
@@ -144,6 +143,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
if req.SectionIndex != nil {
|
if req.SectionIndex != nil {
|
||||||
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
stageInstance = fmt.Sprintf("%s:section-%04d", stageInstance, *req.SectionIndex)
|
||||||
}
|
}
|
||||||
|
if req.RepairProposalIndex != nil && req.RepairAttempt != nil {
|
||||||
|
stageInstance = fmt.Sprintf("%s:proposal-%04d:attempt-%04d", stageInstance, *req.RepairProposalIndex, *req.RepairAttempt)
|
||||||
|
}
|
||||||
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
|
stage := stagename.ValidatorBatch(stageInstance, v.name, batch.BatchIndex)
|
||||||
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
responseSchema := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
|
||||||
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
responsePayload := map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model}
|
||||||
@@ -155,12 +157,13 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
stage,
|
stage,
|
||||||
map[string]any{
|
map[string]any{
|
||||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||||
"validator_name": v.name,
|
"validator_name": v.name,
|
||||||
"validator_type": v.validatorType,
|
"validator_type": v.validatorType,
|
||||||
"batch_index": batch.BatchIndex,
|
"batch_index": batch.BatchIndex,
|
||||||
"proposal_indices": batchProposalIndices(batch.Items),
|
"proposal_indices": batchProposalIndices(batch.Items),
|
||||||
"prompt": completion.Prompt,
|
"repair_proposal_index": req.RepairProposalIndex, "repair_attempt": req.RepairAttempt,
|
||||||
"response_schema": responseSchema.DiagnosticsMap(),
|
"prompt": completion.Prompt,
|
||||||
|
"response_schema": responseSchema.DiagnosticsMap(),
|
||||||
},
|
},
|
||||||
completion.Prepared,
|
completion.Prepared,
|
||||||
responsePayload,
|
responsePayload,
|
||||||
@@ -263,7 +266,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
|||||||
}
|
}
|
||||||
|
|
||||||
seg := segments[position]
|
seg := segments[position]
|
||||||
preview := proposals.PreviewProposalForSegment(&seg, p.CorrectionProposal, req.ReplacementPolicy)
|
preview := previewForEvidence(req, &seg, p.CorrectionProposal)
|
||||||
if !preview.Applicable {
|
if !preview.Applicable {
|
||||||
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
|
immediate = append(immediate, rejection(p.ProposalIndex, string(preview.SkipReason), "proposal is not previewable for LLM validation"))
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -36,6 +36,9 @@ type Request struct {
|
|||||||
StageID *string `json:"stage_id,omitempty"`
|
StageID *string `json:"stage_id,omitempty"`
|
||||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
|
IndependentEvidence bool `json:"-"`
|
||||||
|
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||||
|
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||||
SectionIndex *int `json:"section_index,omitempty"`
|
SectionIndex *int `json:"section_index,omitempty"`
|
||||||
WorkingTranscript *schema.Transcript `json:"-"`
|
WorkingTranscript *schema.Transcript `json:"-"`
|
||||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||||
|
|||||||
Reference in New Issue
Block a user