Remove confidence scoring and refine correction prompts

This commit is contained in:
2026-11-11 17:05:52 +00:00
parent bbf89bb4f5
commit d155bf4cb3
92 changed files with 448 additions and 786 deletions

View File

@@ -63,7 +63,6 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
type deterministicReplacementRule struct {
OriginalText string
CorrectedText string
Confidence float64
}
type deterministicFakeModule struct {
@@ -99,7 +98,6 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
TargetSegmentID: seg.ID,
OriginalText: rule.OriginalText,
CorrectedText: rule.CorrectedText,
Confidence: rule.Confidence,
})
}
}
@@ -116,8 +114,8 @@ func TestChunkProposalMetadataAssociation(t *testing.T) {
key: "fake_rewriter",
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
rules: []deterministicReplacementRule{
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
{OriginalText: "alpha", CorrectedText: "ALPHA"},
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
},
}
@@ -149,9 +147,9 @@ func TestDeterministicProposalIndexAssignment(t *testing.T) {
key: "fake_rewriter",
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
rules: []deterministicReplacementRule{
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE", Confidence: 0.9},
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
{OriginalText: "alpha", CorrectedText: "ALPHA"},
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE"},
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
},
}
@@ -192,9 +190,9 @@ func TestApplyFakeProposalsAcrossMultipleChunksWithStaleSkip(t *testing.T) {
key: "fake_rewriter",
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
rules: []deterministicReplacementRule{
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE", Confidence: 0.9},
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
{OriginalText: "alpha", CorrectedText: "ALPHA"},
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE"},
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
},
}
@@ -233,8 +231,8 @@ func TestChunkProposalCompositionDoesNotMutateNormalizedTranscript(t *testing.T)
key: "fake_rewriter",
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
rules: []deterministicReplacementRule{
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
{OriginalText: "alpha", CorrectedText: "ALPHA"},
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
},
}

View File

@@ -56,7 +56,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal
_ = req
return ProposalResult{
Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b"},
},
}, nil
}

View File

@@ -199,7 +199,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
{name: "empty original", original: "", replacement: `"corrected_text":"x"`, valid: false},
} {
t.Run(tc.name, func(t *testing.T) {
fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original), `"confidence":0.9`}
fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original)}
if tc.replacement != "" {
fields = append(fields, tc.replacement)
}
@@ -427,7 +427,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
}
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)

View File

@@ -67,7 +67,7 @@ prompts:
Messages []struct{ Role, Content string } `json:"messages"`
ResponseFormat map[string]any `json:"response_format"`
}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var request struct {
Model string `json:"model"`
@@ -173,7 +173,7 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
}
var messages [][]wireMessage
var models []string
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b"}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var req struct {
Model string `json:"model"`

View File

@@ -215,7 +215,7 @@ func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
revised.TargetSegmentID = 2
revised.OriginalText = "uh "
revised.CorrectedText = ""
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "original_text_presence", ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "llm_rejected"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
if len(ledger) != 1 {
t.Fatalf("ledger: %+v", ledger)
@@ -233,7 +233,6 @@ func TestSkippedRepairLedgerKeepsApplicationReason(t *testing.T) {
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: instance},
}
revision := original
revision.Confidence = 0.99
skip := proposals.SkippedChange{ProposalIndex: 1, SkipReason: proposals.SkipReasonMissingOriginalText, Message: "source span no longer matches"}
output := &runner.RunOutput{
ModuleResults: []runner.ModuleResult{{ModuleInstance: instance, SkippedChanges: []proposals.SkippedChange{skip}}},
@@ -259,7 +258,7 @@ func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
revision := original
revision.CorrectedText = "fox"
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "original_text_presence", ProposalIndex: 1, ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
var found *CorrectionLedgerEntry
for i := range ledger {

View File

@@ -29,7 +29,7 @@ func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.Transcr
func (m partialModule) Propose(context.Context, contracts.ProposalRequest) (contracts.ProposalResult, error) {
result := contracts.ProposalResult{}
for _, text := range []string{"Hello", "Greetings", "Hi"} {
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text, Confidence: 1})
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text})
}
return result, nil
}
@@ -55,7 +55,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
return structuredoutput.Response{}, nil
}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Confidence: 1, Reason: "completed assessment"}}}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
return structuredoutput.Response{}, nil
}

View File

@@ -27,7 +27,7 @@ func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) {
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
calls := 0
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present"}]}`}
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
output := outputs[len(outputs)-1]
if calls < len(outputs) {
@@ -76,16 +76,12 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if calls != 2 || len(generated.Enriched) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
if missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
t.Fatalf("domain checks or extra semantic retry: missing=%+v calls=%d", missing, calls)
}
}

View File

@@ -35,10 +35,9 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
}
// StructuredCorrectionSet is the reusable structured LLM response model for
@@ -201,7 +200,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
TargetSegmentID: targetID,
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
}
corrections = append(corrections, candidate)

View File

@@ -86,7 +86,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
@@ -121,7 +121,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
req := defaultRequest(t)
@@ -143,7 +143,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
diag := &captureDiagnosticsWriter{}
@@ -175,7 +175,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
},
},
},
@@ -190,7 +190,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if len(result.Corrections) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
@@ -238,8 +238,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -269,8 +269,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
@@ -316,7 +316,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)

View File

@@ -22,7 +22,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
client := &fakeStructuredClient{
completion: structuredoutput.Response{Exchange: exchange},
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
section := contracts.SectionMetadata{Index: 7}
@@ -102,10 +102,10 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: ""},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello"},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the"},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{

View File

@@ -6,7 +6,7 @@ import (
)
func TestRepairRequestJSONPreservesCurrentVersionAndFindings(t *testing.T) {
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: "", Confidence: 0.7}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: ""}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
text, err := request.JSON()
if err != nil {
t.Fatal(err)

View File

@@ -13,7 +13,7 @@ func TestDecodeRepairResponse(t *testing.T) {
name, content string
valid bool
}{
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true},
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},

View File

@@ -15,7 +15,6 @@ func TestApplyProposalsApplyingOneProposal(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.95,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "glossary", ModuleInstance: "glossary_1"},
},
@@ -42,7 +41,6 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "Hrank",
CorrectedText: "X",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 2, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -51,7 +49,6 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "rank",
CorrectedText: "Hrank",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
@@ -78,7 +75,6 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "foo bar",
CorrectedText: "foo-bar",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -87,7 +83,6 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "foo bar",
CorrectedText: "foobar",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -112,8 +107,8 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
func TestApplyProposalsRechecksBlankResultAfterEarlierDeletion(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
proposed := []EnrichedCorrectionProposal{
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "", Confidence: 1}, ProposalMetadata: ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "b", CorrectedText: "", Confidence: 1}, ProposalMetadata: ProposalMetadata{ProposalIndex: 1}},
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "a", CorrectedText: ""}, ProposalMetadata: ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "b", CorrectedText: ""}, ProposalMetadata: ProposalMetadata{ProposalIndex: 1}},
}
for _, candidate := range proposed {
if preview := PreviewProposalForSegment(&transcript.Segments[0], candidate.CorrectionProposal, ReplacementPolicyRequireUnique); !preview.Applicable {
@@ -137,7 +132,6 @@ func TestApplyProposalsMissingSegmentSkip(t *testing.T) {
TargetSegmentID: 999,
OriginalText: "hello",
CorrectedText: "hi",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -161,7 +155,6 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "uh",
CorrectedText: "um",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
},
@@ -185,7 +178,6 @@ func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "uh",
CorrectedText: "um",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
},
@@ -214,7 +206,6 @@ func TestApplyProposalsNoMutationOfInputTranscript(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "hello",
CorrectedText: "hi",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -243,7 +234,6 @@ func TestApplyProposalsSegmentMetadataPreservation(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "rank",
CorrectedText: "Hrank",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "glossary", ModuleInstance: "glossary_1"},
},
@@ -268,7 +258,6 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "hello",
CorrectedText: "hi",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},
@@ -277,7 +266,6 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) {
TargetSegmentID: 999,
OriginalText: "x",
CorrectedText: "y",
Confidence: 0.9,
},
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
},

View File

@@ -13,7 +13,6 @@ func TestPreviewProposalForSegmentRequireUniqueSuccess(t *testing.T) {
TargetSegmentID: 7,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -34,7 +33,6 @@ func TestPreviewProposalForSegmentRequireUniqueMissingOriginalText(t *testing.T)
TargetSegmentID: 7,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -52,7 +50,6 @@ func TestPreviewProposalForSegmentRequireUniqueAmbiguousOriginalText(t *testing.
TargetSegmentID: 7,
OriginalText: "rank",
CorrectedText: "Hrank",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -70,7 +67,6 @@ func TestPreviewProposalForSegmentReplaceAllSuccessMultipleOccurrences(t *testin
TargetSegmentID: 8,
OriginalText: "uh",
CorrectedText: "um",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
@@ -91,7 +87,6 @@ func TestPreviewProposalForSegmentReplaceAllMissingOriginalText(t *testing.T) {
TargetSegmentID: 8,
OriginalText: "uh",
CorrectedText: "um",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
@@ -109,7 +104,6 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
TargetSegmentID: 8,
OriginalText: "hello",
CorrectedText: "hello",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -127,7 +121,6 @@ func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonE
TargetSegmentID: 8,
OriginalText: "uh ",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -140,7 +133,7 @@ func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonE
}
func TestPreviewProposalForSegmentWhitespaceSpan(t *testing.T) {
proposal := CorrectionProposal{TargetSegmentID: 8, OriginalText: " ", CorrectedText: " ", Confidence: 0.9}
proposal := CorrectionProposal{TargetSegmentID: 8, OriginalText: " ", CorrectedText: " "}
unique := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello world"}, proposal, ReplacementPolicyRequireUnique)
if !unique.Applicable || unique.CorrectedSegmentText != "hello world" || unique.ReplacementCount != 1 {
t.Fatalf("unexpected spacing preview: %+v", unique)
@@ -157,7 +150,7 @@ func TestPreviewProposalForSegmentWhitespaceSpan(t *testing.T) {
}
func TestPreviewProposalForSegmentRejectsEmptyOriginalText(t *testing.T) {
preview := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello"}, CorrectionProposal{TargetSegmentID: 8, OriginalText: "", CorrectedText: "x", Confidence: 0.9}, ReplacementPolicyRequireUnique)
preview := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello"}, CorrectionProposal{TargetSegmentID: 8, OriginalText: "", CorrectedText: "x"}, ReplacementPolicyRequireUnique)
if preview.Applicable || preview.SkipReason != SkipReasonInvalidProposal {
t.Fatalf("expected invalid empty original: %+v", preview)
}
@@ -169,7 +162,6 @@ func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
TargetSegmentID: 8,
OriginalText: "uh",
CorrectedText: "",
Confidence: 0.9,
}
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
@@ -188,7 +180,6 @@ func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
TargetSegmentID: 9,
OriginalText: "rank",
CorrectedText: "Hrank",
Confidence: 0.9,
}
_ = PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
@@ -205,7 +196,6 @@ func TestPreviewProposalForSegmentLiteralMatchingCaseSensitiveAndPunctuation(t *
TargetSegmentID: 10,
OriginalText: "world.",
CorrectedText: "earth.",
Confidence: 0.9,
}
punctuationResult := PreviewProposalForSegment(segment, punctuationProposal, ReplacementPolicyRequireUnique)
if !punctuationResult.Applicable {
@@ -219,7 +209,6 @@ func TestPreviewProposalForSegmentLiteralMatchingCaseSensitiveAndPunctuation(t *
TargetSegmentID: 10,
OriginalText: "hello",
CorrectedText: "Hi",
Confidence: 0.9,
}
caseMismatchResult := PreviewProposalForSegment(segment, caseMismatchProposal, ReplacementPolicyRequireUnique)
if caseMismatchResult.Applicable {

View File

@@ -6,10 +6,9 @@ import (
// CorrectionProposal is the base proposal emitted by correction modules.
type CorrectionProposal struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
}
// ProposalMetadata is framework-enriched execution metadata for a proposal.
@@ -37,9 +36,6 @@ func (p CorrectionProposal) Validate() error {
if p.OriginalText == "" {
return fmt.Errorf("proposal original_text must not be empty")
}
if p.Confidence < 0.0 || p.Confidence > 1.0 {
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
}
return nil
}

View File

@@ -10,7 +10,6 @@ func TestCorrectionProposalValidate_Valid(t *testing.T) {
TargetSegmentID: 42,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.95,
}
if err := proposal.Validate(); err != nil {
@@ -23,7 +22,6 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
TargetSegmentID: 42,
OriginalText: "",
CorrectedText: "Jesters",
Confidence: 0.95,
}
err := proposal.Validate()
@@ -40,7 +38,6 @@ func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
TargetSegmentID: 42,
OriginalText: "gestures",
CorrectedText: "",
Confidence: 0.95,
}
if err := proposal.Validate(); err != nil {
@@ -49,35 +46,17 @@ func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
}
func TestCorrectionProposalValidate_AllowsWhitespaceOriginalText(t *testing.T) {
proposal := CorrectionProposal{TargetSegmentID: 42, OriginalText: " ", CorrectedText: " ", Confidence: 0.95}
proposal := CorrectionProposal{TargetSegmentID: 42, OriginalText: " ", CorrectedText: " "}
if err := proposal.Validate(); err != nil {
t.Fatalf("expected whitespace span to be valid, got %v", err)
}
}
func TestCorrectionProposalValidate_InvalidConfidence(t *testing.T) {
proposal := CorrectionProposal{
TargetSegmentID: 42,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 1.1,
}
err := proposal.Validate()
if err == nil {
t.Fatal("expected validation error, got nil")
}
if err.Error() != "proposal confidence must be between 0.0 and 1.0" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCorrectionProposalValidate_InvalidSegmentID(t *testing.T) {
proposal := CorrectionProposal{
TargetSegmentID: 0,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.95,
}
err := proposal.Validate()
@@ -147,7 +126,6 @@ func TestEnrichedCorrectionProposalJSONIncludesMetadata(t *testing.T) {
TargetSegmentID: 7,
OriginalText: "teh",
CorrectedText: "the",
Confidence: 0.8,
},
ProposalMetadata: ProposalMetadata{
ProposalIndex: 1,
@@ -162,7 +140,7 @@ func TestEnrichedCorrectionProposalJSONIncludesMetadata(t *testing.T) {
t.Fatalf("marshal failed: %v", err)
}
const expected = `{"id":7,"original_text":"teh","corrected_text":"the","confidence":0.8,"proposal_index":1,"module_key":"grammar","module_instance":"grammar_1","section_index":3}`
const expected = `{"id":7,"original_text":"teh","corrected_text":"the","proposal_index":1,"module_key":"grammar","module_instance":"grammar_1","section_index":3}`
if string(b) != expected {
t.Fatalf("unexpected json\nexpected: %s\nactual: %s", expected, string(b))
}

View File

@@ -18,8 +18,7 @@
"module_instance": "homophones",
"id": 1,
"original_text": "rank",
"corrected_text": "Hrank",
"confidence": 0.9
"corrected_text": "Hrank"
}
]
}

View File

@@ -27,8 +27,7 @@
"module_instance": "glossary_1",
"id": 2,
"original_text": "rank",
"corrected_text": "Hrank",
"confidence": 0.9
"corrected_text": "Hrank"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "grammar_1",
"id": 99,
"original_text": "hello",
"corrected_text": "hi",
"confidence": 0.9
"corrected_text": "hi"
}
]
}

View File

@@ -25,8 +25,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "foo bar",
"corrected_text": "foo-bar",
"confidence": 0.9
"corrected_text": "foo-bar"
},
{
"proposal_index": 1,
@@ -34,8 +33,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "foo bar",
"corrected_text": "foobar",
"confidence": 0.9
"corrected_text": "foobar"
},
{
"proposal_index": 2,
@@ -43,8 +41,7 @@
"module_instance": "homophones",
"id": 1,
"original_text": "rank",
"corrected_text": "Hrank",
"confidence": 0.9
"corrected_text": "Hrank"
},
{
"proposal_index": 3,
@@ -52,8 +49,7 @@
"module_instance": "grammar_1",
"id": 99,
"original_text": "x",
"corrected_text": "y",
"confidence": 0.9
"corrected_text": "y"
},
{
"proposal_index": 4,
@@ -61,8 +57,7 @@
"module_instance": "spoken_word",
"id": 2,
"original_text": "hello",
"corrected_text": "hello",
"confidence": 0.9
"corrected_text": "hello"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "bar",
"corrected_text": "BAR",
"confidence": 0.9
"corrected_text": "BAR"
},
{
"proposal_index": 1,
@@ -27,8 +26,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "foo",
"corrected_text": "FOO",
"confidence": 0.9
"corrected_text": "FOO"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "spoken_word",
"id": 1,
"original_text": "uh",
"corrected_text": "um",
"confidence": 0.9
"corrected_text": "um"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "spoken_word",
"id": 1,
"original_text": "ha",
"corrected_text": "ho",
"confidence": 0.9
"corrected_text": "ho"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "glossary_1",
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95
"corrected_text": "Jesters"
}
]
}

View File

@@ -18,8 +18,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "foo bar",
"corrected_text": "foo-bar",
"confidence": 0.9
"corrected_text": "foo-bar"
},
{
"proposal_index": 1,
@@ -27,8 +26,7 @@
"module_instance": "grammar_1",
"id": 1,
"original_text": "foo bar",
"corrected_text": "foobar",
"confidence": 0.9
"corrected_text": "foobar"
}
]
}

View File

@@ -26,8 +26,7 @@
"required": [
"id",
"original_text",
"corrected_text",
"confidence"
"corrected_text"
],
"properties": {
"id": {
@@ -40,11 +39,6 @@
},
"corrected_text": {
"type": "string"
},
"confidence": {
"type": "number",
"minimum": 0,
"maximum": 1
}
}
},

View File

@@ -13,8 +13,7 @@
"required": [
"id",
"original_text",
"corrected_text",
"confidence"
"corrected_text"
],
"properties": {
"id": {
@@ -27,11 +26,6 @@
},
"corrected_text": {
"type": "string"
},
"confidence": {
"type": "number",
"minimum": 0,
"maximum": 1
}
}
}

View File

@@ -13,7 +13,6 @@
"required": [
"correction_index",
"approved",
"confidence",
"reason"
],
"properties": {
@@ -24,11 +23,6 @@
"approved": {
"type": "boolean"
},
"confidence": {
"type": "number",
"minimum": 0,
"maximum": 1
},
"reason": {
"type": "string"
}

View File

@@ -76,9 +76,6 @@ func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
if !json.Valid(schema.JSONSchema) {
t.Fatalf("expected valid JSON schema for %q", schema.ID)
}
if !bytes.Contains(schema.JSONSchema, []byte("\n ")) {
t.Fatalf("expected readable formatted JSON schema for %q", schema.ID)
}
}
}
@@ -152,14 +149,13 @@ func TestCorrectionRepairSchema(t *testing.T) {
name, content string
valid bool
}{
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0}}`, true},
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
{"withdraw", `{"proposal_ref":1,"action":"withdraw","correction":null}`, true},
{"missing correction", `{"proposal_ref":1,"action":"withdraw"}`, false},
{"extra field", `{"proposal_ref":1,"action":"withdraw","correction":null,"extra":1}`, false},
{"zero reference", `{"proposal_ref":0,"action":"withdraw","correction":null}`, false},
{"unknown action", `{"proposal_ref":1,"action":"other","correction":null}`, false},
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x","confidence":1}}`, false},
{"confidence outside range", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"x","corrected_text":"y","confidence":2}}`, false},
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x"}}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
var value any

View File

@@ -105,7 +105,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
}}
sections := []chunking.Section{}
for i := 0; i < 3; i++ {

View File

@@ -1,8 +1,6 @@
package runner
import (
"math"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -20,15 +18,12 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract
}
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
var applicable bool
switch validator.Name() {
case "proposal_shape":
applicable = true
case "no_effect":
applicable = usableSpan
case "confidence_threshold":
applicable = confidence
case "original_text_presence":
applicable = usableSpan && target
case "protected_terms":

View File

@@ -17,8 +17,8 @@ import (
)
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
candidate := func(index, target int, original, replacement string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
}
var seen []int
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -29,7 +29,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
return validators.Result{Decisions: decisions}, nil
}}, validatormetadata.ExecutionClassLLMBacked)
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -41,7 +41,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
reasons[d.ProposalIndex][d.ReasonCode] = true
}
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
if !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
t.Fatalf("findings = %+v", reasons)
}
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
@@ -50,7 +50,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
var later int
@@ -82,7 +82,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
}
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
rejected := []ValidatorRejectedChange{{ValidatorName: "editorial_review", ProposalIndex: 2, ReasonCode: "llm_rejected", Message: "The proposed replacement is unsupported by the supplied context."}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
findings := repairFindings(2, rejected)
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
t.Fatalf("findings: %+v", findings)
@@ -91,8 +91,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
@@ -102,7 +102,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
attempt := 1
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -123,29 +123,12 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
}
}
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
t.Fatalf("result: %+v", result)
}
}
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
identity := glossary.Identity{Collection: "party", ID: "jesters"}
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.ValidateProtectedTerms(req)
}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {

View File

@@ -37,7 +37,7 @@ func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest)
if m.spec.Index == 2 {
original, corrected = "Party Spell", "Party Spell!"
}
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected, Confidence: 1}}}, nil
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected}}}, nil
}
func TestOrderedWorkflowUsesIndependentSelectionsAndPriorEdits(t *testing.T) {

View File

@@ -119,7 +119,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
findings := safeFindings(rejected)
replay := &structuredoutput.Replay{Exchange: input.Exchange}
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText, Confidence: current.Confidence}, Findings: findings}).JSON()
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
if err != nil {
branches = append(branches, branch)
return finish(err)
@@ -198,7 +198,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
target = input.WorkingTranscript.Segments[localRef-1].ID
}
current = original
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
if debug {
branch.Final = current
}

View File

@@ -46,12 +46,12 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -82,15 +82,15 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
calls int
}{
{"withdraw", []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}, nil, 1, "withdrawn", 1},
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.1}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.2}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat","confidence":0.9}}`}, nil, 2, "rejected", 1},
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"action":"withdraw","correction":null}`}, nil, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -114,15 +114,15 @@ type repairModule struct{ outcomeModule }
func (m repairModule) RepairPromptID() string { return "repair" }
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl", Confidence: .1}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox","confidence":0.9}}`,
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl","confidence":0.9}}`,
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
if err != nil {
t.Fatal(err)
}
@@ -142,8 +142,8 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
for _, tc := range []struct {
name string
@@ -159,11 +159,11 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}
return validators.Result{Decisions: decisions}, nil
}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl","confidence":0.9}}`}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -185,8 +185,8 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -200,10 +200,10 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -237,8 +237,8 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
t.Fatal(err)
}
warnings := 0
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -267,7 +267,7 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
validation := &repairClient{secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
@@ -285,3 +285,24 @@ func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
t.Fatalf("credential in artifact: %s", data)
}
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
approved := req.RepairAttempt != nil
for _, index := range preserved {
if p.ProposalIndex == index {
approved = true
}
}
disposition, reason := validators.DispositionRejected, "llm_rejected"
if approved {
disposition, reason = validators.DispositionApproved, validators.ReasonApproved
}
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: reason, Message: "The proposed replacement is unsupported by the supplied context."})
}
return validators.Result{Decisions: decisions}, nil
}}
}

View File

@@ -13,10 +13,8 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -57,7 +55,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
}}
second := fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"first": failed, "second": second}})
out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {ModuleKey: "second", InstanceName: "second"}}})
@@ -94,9 +92,9 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired}}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
@@ -119,7 +117,7 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
origin := errors.New("validator setup failed")
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
}}
@@ -133,7 +131,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}
@@ -236,7 +234,7 @@ func TestRunnerOneModuleAppliesProposal(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
@@ -256,13 +254,13 @@ func TestRunnerModulesRunSequentially(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if req.WorkingTranscript.Segments[0].Text != "the cat" {
t.Fatalf("second module did not see first module changes: %q", req.WorkingTranscript.Segments[0].Text)
}
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}},
}})
@@ -293,7 +291,7 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
if req.Section.Index != 0 {
return nil, nil
}
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText, Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText}}, nil
}},
"inspect": fakeModule{key: "inspect", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
ids := make([]int, len(req.WorkingTranscript.Segments))
@@ -307,7 +305,7 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
return nil, errors.New("later module did not see updated section text")
}
if req.Section.Index == 0 {
return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C"}}, nil
}
return nil, nil
}},
@@ -449,7 +447,7 @@ func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *tes
started <- seg.ID
<-releaseByID[seg.ID]
return []proposals.CorrectionProposal{
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
}, nil
}},
}})
@@ -584,7 +582,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
<-releaseSectionOne
}
return []proposals.CorrectionProposal{
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
}, nil
},
},
@@ -630,7 +628,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
},
}
}
@@ -662,7 +660,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -672,7 +670,7 @@ func TestRunnerSkippedRecorded(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
@@ -688,7 +686,7 @@ func TestRunnerFailureReturnsPartialProgress(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return nil, errors.New("boom")
@@ -712,7 +710,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
before := transcript.Segments[0].Text
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
@@ -747,7 +745,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{allowAll},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
},
},
}})
@@ -775,7 +773,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
@@ -813,8 +811,8 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
}, nil
}},
}})
@@ -831,14 +829,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}}
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
// Missing one decision triggers cardinality error.
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
}}
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
@@ -861,7 +859,7 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
@@ -884,8 +882,8 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "b", CorrectedText: "", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: ""},
{TargetSegmentID: 1, OriginalText: "b", CorrectedText: ""},
},
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
}, nil
@@ -953,7 +951,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
@@ -961,7 +959,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
cfg := runnerTestConfig()
@@ -979,11 +977,11 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
cfg := runnerTestConfig()
@@ -1004,14 +1002,14 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}},
}})
cfg := runnerTestConfig()
@@ -1036,9 +1034,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
@@ -1057,9 +1055,9 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good"},
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
@@ -1086,7 +1084,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
cfg := runnerTestConfig()
@@ -1102,13 +1100,13 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
cfg := runnerTestConfig()
@@ -1128,8 +1126,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
llmValidator.SetTokenEstimator(lenEstimator{})
@@ -1139,8 +1137,8 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1},
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
}, nil
}},
}})
@@ -1164,7 +1162,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
@@ -1172,7 +1170,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
out, err := r.Run(context.Background(), RunInput{
@@ -1262,7 +1260,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
@@ -1278,7 +1276,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
ProposalDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1299,70 +1297,6 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
}
}
func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
},
},
},
}
cfg := runnerTestConfig()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "hello ,world" {
t.Fatalf("expected no changes due to confidence threshold, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 {
t.Fatalf("expected validator rejection, got %+v", out.ModuleResults)
}
if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence {
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
}
}
func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
},
},
},
}
cfg := runnerTestConfig()
cfg.Thresholds.Glossary = 0.9
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "There were gestures" {
t.Fatalf("expected no changes due to glossary confidence threshold, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 {
t.Fatalf("expected validator rejection, got %+v", out.ModuleResults)
}
if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence {
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
}
}
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
origin := errors.New("original fatal failure")
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}

View File

@@ -2,7 +2,6 @@ package validators
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
@@ -21,8 +20,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
case c.OriginalText == "":
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
case c.Confidence < 0.0 || c.Confidence > 1.0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
default:
decisions = append(decisions, approval(c.ProposalIndex))
}
@@ -33,26 +30,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
threshold := req.Settings.ConfidenceThreshold
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if c.Confidence < threshold {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonLowConfidence, fmt.Sprintf("confidence %.4f below threshold %.4f", c.Confidence, threshold)))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type OriginalTextPresenceValidator struct{}
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }

View File

@@ -11,7 +11,7 @@ func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) {
switch reason {
case ReasonInvalidTargetSegment, ReasonMissingTargetSegment,
ReasonEmptyOriginalText, ReasonMissingOriginalText,
ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText,
ReasonNoEffect, ReasonEmptyResultingText,
string(proposals.SkipReasonAmbiguousOriginal),
string(proposals.SkipReasonInvalidProposal):
return outcomes.ReasonCode(reason), true

View File

@@ -5,14 +5,14 @@ import "testing"
func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) {
for _, reason := range []string{
ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText,
ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect,
ReasonMissingOriginalText, ReasonNoEffect,
ReasonEmptyResultingText, "ambiguous_original_text",
} {
if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason {
t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok)
}
}
for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
for _, reason := range []string{ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
if got, ok := DefectiveProposalReason(reason); ok {
t.Fatalf("ordinary rejection %q classified as %q", reason, got)
}

View File

@@ -23,7 +23,7 @@ type validationTransport func(*http.Request) (*http.Response, error)
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
valid := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string

View File

@@ -43,10 +43,9 @@ type LLMValidationRequest struct {
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Confidence float64 `json:"confidence"`
Reason string `json:"reason"`
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
}
type LLMValidationResponse struct {

View File

@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -319,11 +319,6 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
continue
}
d := entries[0]
if d.Confidence < 0.0 || d.Confidence > 1.0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
reasonCode := ReasonApproved
if !d.Approved {
reasonCode = "llm_rejected"

View File

@@ -72,7 +72,7 @@ func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
}
}
@@ -151,8 +151,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Reason: ""},
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -164,7 +164,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
if err != nil {
t.Fatalf("validate err: %v", err)
}
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved || res.Decisions[0].Message != "" || res.Decisions[1].Message != "bad" {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(client.calls) != 1 {
@@ -206,10 +206,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
approve := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
}
reject := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
}
type expectedDecision struct {
index int
@@ -230,7 +230,6 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -245,7 +244,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
}
for i, want := range tc.want {
got := result.Decisions[i]
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision) {
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
}
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
@@ -289,7 +288,7 @@ func equalIndices(a, b []int) bool {
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
huge := strings.Repeat("gestures ", 200)
@@ -300,11 +299,11 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
@@ -335,7 +334,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -374,7 +373,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -399,7 +398,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }

View File

@@ -14,7 +14,6 @@ import (
const (
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyResultingText = "empty_resulting_segment"
@@ -22,12 +21,10 @@ const (
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large"
ReasonMissingValidationDecision = "missing_validation_decision"
ReasonDuplicateValidationDecision = "duplicate_validation_decision"
ReasonInvalidValidationDecision = "invalid_validation_decision"
)
// Request is the runtime input shared by deterministic validators.

View File

@@ -39,12 +39,11 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "Jesters",
CorrectedText: "Gestures",
Confidence: 0.9,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1},
},
},
@@ -68,7 +67,6 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.9,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
},
@@ -99,7 +97,7 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}},
}
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
@@ -120,8 +118,7 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mira", Name: "Mira", Aliases: []string{"Captain Mira"}}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain", Confidence: 0.9,
},
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain"},
}},
}
result, err := ValidateProtectedTerms(req)

View File

@@ -1,8 +1,6 @@
package validators
import (
"math"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -50,9 +48,6 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
segment = &section.segments[position]
}
correction := candidate.CorrectionProposal
if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy)
if r.Previews != nil {
r.Previews[candidate.ProposalIndex] = preview

View File

@@ -5,21 +5,20 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
}
}
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
}
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "missing decision") {
t.Fatalf("expected missing decision error, got %v", err)
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
t.Fatalf("expected duplicate decision error, got %v", err)
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
t.Fatalf("expected unknown decision error, got %v", err)
}
}
func TestConfidenceThresholdValidator(t *testing.T) {
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
}
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
}
}
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, "", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
mkCandidate(4, 1, " ", " ", 0.9),
mkCandidate(0, 1, "teh", "the"),
mkCandidate(1, 0, "teh", "the"),
mkCandidate(2, 1, "", "the"),
mkCandidate(4, 1, " ", " "),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
if !res.Decisions[4].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
if !res.Decisions[3].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "missing", "x", 0.9),
mkCandidate(2, 5, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "missing", "x"),
mkCandidate(2, 5, "hello", "hi"),
}}
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
mkCandidate(2, 1, "hello ", "", 0.9),
mkCandidate(3, 1, "hello world", "", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "hello world", " "),
mkCandidate(2, 1, "hello ", ""),
mkCandidate(3, 1, "hello world", ""),
}}
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
func TestNoEffectValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hello", 0.9),
mkCandidate(1, 1, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hello"),
mkCandidate(1, 1, "hello", "hi"),
}}
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),
},
}
res, err := ValidateProtectedTerms(req)
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
func TestStableReasonCodes(t *testing.T) {
codes := []string{
ReasonApproved,
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyResultingText,
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
}

View File

@@ -50,8 +50,8 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
@@ -89,8 +89,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
terms := glossary.Vocabulary(protected, exclusions)
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1,
}}
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}
policy := proposals.ReplacementPolicyRequireUnique
decision, err := ValidateProtectedTerms(Request{ModuleKey: tc.module, WorkingTranscript: transcript, ReplacementPolicy: policy,
ProtectedTerms: terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{candidate}})