Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -63,7 +63,6 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
|
||||
type deterministicReplacementRule struct {
|
||||
OriginalText string
|
||||
CorrectedText string
|
||||
Confidence float64
|
||||
}
|
||||
|
||||
type deterministicFakeModule struct {
|
||||
@@ -99,7 +98,6 @@ func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalReques
|
||||
TargetSegmentID: seg.ID,
|
||||
OriginalText: rule.OriginalText,
|
||||
CorrectedText: rule.CorrectedText,
|
||||
Confidence: rule.Confidence,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -116,8 +114,8 @@ func TestChunkProposalMetadataAssociation(t *testing.T) {
|
||||
key: "fake_rewriter",
|
||||
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
rules: []deterministicReplacementRule{
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -149,9 +147,9 @@ func TestDeterministicProposalIndexAssignment(t *testing.T) {
|
||||
key: "fake_rewriter",
|
||||
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
rules: []deterministicReplacementRule{
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
|
||||
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE", Confidence: 0.9},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE"},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -192,9 +190,9 @@ func TestApplyFakeProposalsAcrossMultipleChunksWithStaleSkip(t *testing.T) {
|
||||
key: "fake_rewriter",
|
||||
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
rules: []deterministicReplacementRule{
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
|
||||
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE", Confidence: 0.9},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
{OriginalText: "alpha one", CorrectedText: "ALPHA-ONE"},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -233,8 +231,8 @@ func TestChunkProposalCompositionDoesNotMutateNormalizedTranscript(t *testing.T)
|
||||
key: "fake_rewriter",
|
||||
replacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
rules: []deterministicReplacementRule{
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9},
|
||||
{OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
{OriginalText: "charlie", CorrectedText: "CHARLIE"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal
|
||||
_ = req
|
||||
return ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b", Confidence: 0.9},
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "b"},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
|
||||
{name: "empty original", original: "", replacement: `"corrected_text":"x"`, valid: false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original), `"confidence":0.9`}
|
||||
fields := []string{`"id":1`, `"original_text":` + fmt.Sprintf("%q", tc.original)}
|
||||
if tc.replacement != "" {
|
||||
fields = append(fields, tc.replacement)
|
||||
}
|
||||
@@ -427,7 +427,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
|
||||
}
|
||||
|
||||
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"reason":"ok"}]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -67,7 +67,7 @@ prompts:
|
||||
Messages []struct{ Role, Content string } `json:"messages"`
|
||||
ResponseFormat map[string]any `json:"response_format"`
|
||||
}
|
||||
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
@@ -173,7 +173,7 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
|
||||
}
|
||||
var messages [][]wireMessage
|
||||
var models []string
|
||||
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b"}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
var req struct {
|
||||
Model string `json:"model"`
|
||||
|
||||
@@ -215,7 +215,7 @@ func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
|
||||
revised.TargetSegmentID = 2
|
||||
revised.OriginalText = "uh "
|
||||
revised.CorrectedText = ""
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "original_text_presence", ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "llm_rejected"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("ledger: %+v", ledger)
|
||||
@@ -233,7 +233,6 @@ func TestSkippedRepairLedgerKeepsApplicationReason(t *testing.T) {
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: instance},
|
||||
}
|
||||
revision := original
|
||||
revision.Confidence = 0.99
|
||||
skip := proposals.SkippedChange{ProposalIndex: 1, SkipReason: proposals.SkipReasonMissingOriginalText, Message: "source span no longer matches"}
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{{ModuleInstance: instance, SkippedChanges: []proposals.SkippedChange{skip}}},
|
||||
@@ -259,7 +258,7 @@ func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
|
||||
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
|
||||
revision := original
|
||||
revision.CorrectedText = "fox"
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "original_text_presence", ProposalIndex: 1, ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
var found *CorrectionLedgerEntry
|
||||
for i := range ledger {
|
||||
|
||||
@@ -29,7 +29,7 @@ func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.Transcr
|
||||
func (m partialModule) Propose(context.Context, contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result := contracts.ProposalResult{}
|
||||
for _, text := range []string{"Hello", "Greetings", "Hi"} {
|
||||
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text, Confidence: 1})
|
||||
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Confidence: 1, Reason: "completed assessment"}}}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
|
||||
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
calls := 0
|
||||
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
|
||||
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present"}]}`}
|
||||
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
|
||||
output := outputs[len(outputs)-1]
|
||||
if calls < len(outputs) {
|
||||
@@ -76,16 +76,12 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
if calls != 2 || len(generated.Enriched) != 1 {
|
||||
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
|
||||
t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
|
||||
if missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
|
||||
t.Fatalf("domain checks or extra semantic retry: missing=%+v calls=%d", missing, calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,10 +35,9 @@ type InteractionArtifacts struct {
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
SegmentReference int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
SegmentReference int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionSet is the reusable structured LLM response model for
|
||||
@@ -201,7 +200,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
TargetSegmentID: targetID,
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -121,7 +121,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
@@ -143,7 +143,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
diag := &captureDiagnosticsWriter{}
|
||||
@@ -175,7 +175,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
|
||||
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -190,7 +190,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
if len(result.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
|
||||
if result.Corrections[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
@@ -238,8 +238,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
@@ -269,8 +269,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
@@ -316,7 +316,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
validationSecret := "proposal-validation-secret"
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
|
||||
},
|
||||
}
|
||||
t.Setenv("PRIMARY_SECRET", primarySecret)
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
client := &fakeStructuredClient{
|
||||
completion: structuredoutput.Response{Exchange: exchange},
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 7}
|
||||
@@ -102,10 +102,10 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
|
||||
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
|
||||
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
|
||||
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: ""},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello"},
|
||||
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the"},
|
||||
}}}}
|
||||
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
ProposalRequest: contracts.ProposalRequest{
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
)
|
||||
|
||||
func TestRepairRequestJSONPreservesCurrentVersionAndFindings(t *testing.T) {
|
||||
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: "", Confidence: 0.7}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
|
||||
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: ""}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
|
||||
text, err := request.JSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestDecodeRepairResponse(t *testing.T) {
|
||||
name, content string
|
||||
valid bool
|
||||
}{
|
||||
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true},
|
||||
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
|
||||
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
|
||||
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
|
||||
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},
|
||||
|
||||
@@ -15,7 +15,6 @@ func TestApplyProposalsApplyingOneProposal(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.95,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "glossary", ModuleInstance: "glossary_1"},
|
||||
},
|
||||
@@ -42,7 +41,6 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "Hrank",
|
||||
CorrectedText: "X",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 2, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -51,7 +49,6 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "rank",
|
||||
CorrectedText: "Hrank",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
@@ -78,7 +75,6 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "foo bar",
|
||||
CorrectedText: "foo-bar",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -87,7 +83,6 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "foo bar",
|
||||
CorrectedText: "foobar",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -112,8 +107,8 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
|
||||
func TestApplyProposalsRechecksBlankResultAfterEarlierDeletion(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
|
||||
proposed := []EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "", Confidence: 1}, ProposalMetadata: ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "b", CorrectedText: "", Confidence: 1}, ProposalMetadata: ProposalMetadata{ProposalIndex: 1}},
|
||||
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "a", CorrectedText: ""}, ProposalMetadata: ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: CorrectionProposal{TargetSegmentID: 1, OriginalText: "b", CorrectedText: ""}, ProposalMetadata: ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
for _, candidate := range proposed {
|
||||
if preview := PreviewProposalForSegment(&transcript.Segments[0], candidate.CorrectionProposal, ReplacementPolicyRequireUnique); !preview.Applicable {
|
||||
@@ -137,7 +132,6 @@ func TestApplyProposalsMissingSegmentSkip(t *testing.T) {
|
||||
TargetSegmentID: 999,
|
||||
OriginalText: "hello",
|
||||
CorrectedText: "hi",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -161,7 +155,6 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "um",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
|
||||
},
|
||||
@@ -185,7 +178,6 @@ func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "um",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
|
||||
},
|
||||
@@ -214,7 +206,6 @@ func TestApplyProposalsNoMutationOfInputTranscript(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "hello",
|
||||
CorrectedText: "hi",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -243,7 +234,6 @@ func TestApplyProposalsSegmentMetadataPreservation(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "rank",
|
||||
CorrectedText: "Hrank",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "glossary", ModuleInstance: "glossary_1"},
|
||||
},
|
||||
@@ -268,7 +258,6 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "hello",
|
||||
CorrectedText: "hi",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
@@ -277,7 +266,6 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) {
|
||||
TargetSegmentID: 999,
|
||||
OriginalText: "x",
|
||||
CorrectedText: "y",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
},
|
||||
|
||||
@@ -13,7 +13,6 @@ func TestPreviewProposalForSegmentRequireUniqueSuccess(t *testing.T) {
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -34,7 +33,6 @@ func TestPreviewProposalForSegmentRequireUniqueMissingOriginalText(t *testing.T)
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -52,7 +50,6 @@ func TestPreviewProposalForSegmentRequireUniqueAmbiguousOriginalText(t *testing.
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "rank",
|
||||
CorrectedText: "Hrank",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -70,7 +67,6 @@ func TestPreviewProposalForSegmentReplaceAllSuccessMultipleOccurrences(t *testin
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "um",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
|
||||
@@ -91,7 +87,6 @@ func TestPreviewProposalForSegmentReplaceAllMissingOriginalText(t *testing.T) {
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "um",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
|
||||
@@ -109,7 +104,6 @@ func TestPreviewProposalForSegmentNoEffectReplacement(t *testing.T) {
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "hello",
|
||||
CorrectedText: "hello",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -127,7 +121,6 @@ func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonE
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh ",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -140,7 +133,7 @@ func TestPreviewProposalForSegmentAllowsEmptyCorrectedTextWhenSegmentRemainsNonE
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentWhitespaceSpan(t *testing.T) {
|
||||
proposal := CorrectionProposal{TargetSegmentID: 8, OriginalText: " ", CorrectedText: " ", Confidence: 0.9}
|
||||
proposal := CorrectionProposal{TargetSegmentID: 8, OriginalText: " ", CorrectedText: " "}
|
||||
unique := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello world"}, proposal, ReplacementPolicyRequireUnique)
|
||||
if !unique.Applicable || unique.CorrectedSegmentText != "hello world" || unique.ReplacementCount != 1 {
|
||||
t.Fatalf("unexpected spacing preview: %+v", unique)
|
||||
@@ -157,7 +150,7 @@ func TestPreviewProposalForSegmentWhitespaceSpan(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPreviewProposalForSegmentRejectsEmptyOriginalText(t *testing.T) {
|
||||
preview := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello"}, CorrectionProposal{TargetSegmentID: 8, OriginalText: "", CorrectedText: "x", Confidence: 0.9}, ReplacementPolicyRequireUnique)
|
||||
preview := PreviewProposalForSegment(&schema.Segment{ID: 8, Text: "hello"}, CorrectionProposal{TargetSegmentID: 8, OriginalText: "", CorrectedText: "x"}, ReplacementPolicyRequireUnique)
|
||||
if preview.Applicable || preview.SkipReason != SkipReasonInvalidProposal {
|
||||
t.Fatalf("expected invalid empty original: %+v", preview)
|
||||
}
|
||||
@@ -169,7 +162,6 @@ func TestPreviewProposalForSegmentRejectsEmptyResultingSegment(t *testing.T) {
|
||||
TargetSegmentID: 8,
|
||||
OriginalText: "uh",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
result := PreviewProposalForSegment(segment, proposal, ReplacementPolicyRequireUnique)
|
||||
@@ -188,7 +180,6 @@ func TestPreviewProposalForSegmentPreservesInputSegment(t *testing.T) {
|
||||
TargetSegmentID: 9,
|
||||
OriginalText: "rank",
|
||||
CorrectedText: "Hrank",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
|
||||
_ = PreviewProposalForSegment(segment, proposal, ReplacementPolicyReplaceAll)
|
||||
@@ -205,7 +196,6 @@ func TestPreviewProposalForSegmentLiteralMatchingCaseSensitiveAndPunctuation(t *
|
||||
TargetSegmentID: 10,
|
||||
OriginalText: "world.",
|
||||
CorrectedText: "earth.",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
punctuationResult := PreviewProposalForSegment(segment, punctuationProposal, ReplacementPolicyRequireUnique)
|
||||
if !punctuationResult.Applicable {
|
||||
@@ -219,7 +209,6 @@ func TestPreviewProposalForSegmentLiteralMatchingCaseSensitiveAndPunctuation(t *
|
||||
TargetSegmentID: 10,
|
||||
OriginalText: "hello",
|
||||
CorrectedText: "Hi",
|
||||
Confidence: 0.9,
|
||||
}
|
||||
caseMismatchResult := PreviewProposalForSegment(segment, caseMismatchProposal, ReplacementPolicyRequireUnique)
|
||||
if caseMismatchResult.Applicable {
|
||||
|
||||
@@ -6,10 +6,9 @@ import (
|
||||
|
||||
// CorrectionProposal is the base proposal emitted by correction modules.
|
||||
type CorrectionProposal struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
}
|
||||
|
||||
// ProposalMetadata is framework-enriched execution metadata for a proposal.
|
||||
@@ -37,9 +36,6 @@ func (p CorrectionProposal) Validate() error {
|
||||
if p.OriginalText == "" {
|
||||
return fmt.Errorf("proposal original_text must not be empty")
|
||||
}
|
||||
if p.Confidence < 0.0 || p.Confidence > 1.0 {
|
||||
return fmt.Errorf("proposal confidence must be between 0.0 and 1.0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ func TestCorrectionProposalValidate_Valid(t *testing.T) {
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
if err := proposal.Validate(); err != nil {
|
||||
@@ -23,7 +22,6 @@ func TestCorrectionProposalValidate_InvalidEmptyOriginalText(t *testing.T) {
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
@@ -40,7 +38,6 @@ func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "",
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
if err := proposal.Validate(); err != nil {
|
||||
@@ -49,35 +46,17 @@ func TestCorrectionProposalValidate_AllowsEmptyCorrectedText(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_AllowsWhitespaceOriginalText(t *testing.T) {
|
||||
proposal := CorrectionProposal{TargetSegmentID: 42, OriginalText: " ", CorrectedText: " ", Confidence: 0.95}
|
||||
proposal := CorrectionProposal{TargetSegmentID: 42, OriginalText: " ", CorrectedText: " "}
|
||||
if err := proposal.Validate(); err != nil {
|
||||
t.Fatalf("expected whitespace span to be valid, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_InvalidConfidence(t *testing.T) {
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 42,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 1.1,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error, got nil")
|
||||
}
|
||||
if err.Error() != "proposal confidence must be between 0.0 and 1.0" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionProposalValidate_InvalidSegmentID(t *testing.T) {
|
||||
proposal := CorrectionProposal{
|
||||
TargetSegmentID: 0,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.95,
|
||||
}
|
||||
|
||||
err := proposal.Validate()
|
||||
@@ -147,7 +126,6 @@ func TestEnrichedCorrectionProposalJSONIncludesMetadata(t *testing.T) {
|
||||
TargetSegmentID: 7,
|
||||
OriginalText: "teh",
|
||||
CorrectedText: "the",
|
||||
Confidence: 0.8,
|
||||
},
|
||||
ProposalMetadata: ProposalMetadata{
|
||||
ProposalIndex: 1,
|
||||
@@ -162,7 +140,7 @@ func TestEnrichedCorrectionProposalJSONIncludesMetadata(t *testing.T) {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
const expected = `{"id":7,"original_text":"teh","corrected_text":"the","confidence":0.8,"proposal_index":1,"module_key":"grammar","module_instance":"grammar_1","section_index":3}`
|
||||
const expected = `{"id":7,"original_text":"teh","corrected_text":"the","proposal_index":1,"module_key":"grammar","module_instance":"grammar_1","section_index":3}`
|
||||
if string(b) != expected {
|
||||
t.Fatalf("unexpected json\nexpected: %s\nactual: %s", expected, string(b))
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "homophones",
|
||||
"id": 1,
|
||||
"original_text": "rank",
|
||||
"corrected_text": "Hrank",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "Hrank"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -27,8 +27,7 @@
|
||||
"module_instance": "glossary_1",
|
||||
"id": 2,
|
||||
"original_text": "rank",
|
||||
"corrected_text": "Hrank",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "Hrank"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 99,
|
||||
"original_text": "hello",
|
||||
"corrected_text": "hi",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "hi"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -25,8 +25,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "foo bar",
|
||||
"corrected_text": "foo-bar",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "foo-bar"
|
||||
},
|
||||
{
|
||||
"proposal_index": 1,
|
||||
@@ -34,8 +33,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "foo bar",
|
||||
"corrected_text": "foobar",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "foobar"
|
||||
},
|
||||
{
|
||||
"proposal_index": 2,
|
||||
@@ -43,8 +41,7 @@
|
||||
"module_instance": "homophones",
|
||||
"id": 1,
|
||||
"original_text": "rank",
|
||||
"corrected_text": "Hrank",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "Hrank"
|
||||
},
|
||||
{
|
||||
"proposal_index": 3,
|
||||
@@ -52,8 +49,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 99,
|
||||
"original_text": "x",
|
||||
"corrected_text": "y",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "y"
|
||||
},
|
||||
{
|
||||
"proposal_index": 4,
|
||||
@@ -61,8 +57,7 @@
|
||||
"module_instance": "spoken_word",
|
||||
"id": 2,
|
||||
"original_text": "hello",
|
||||
"corrected_text": "hello",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "hello"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "bar",
|
||||
"corrected_text": "BAR",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "BAR"
|
||||
},
|
||||
{
|
||||
"proposal_index": 1,
|
||||
@@ -27,8 +26,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "foo",
|
||||
"corrected_text": "FOO",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "FOO"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "spoken_word",
|
||||
"id": 1,
|
||||
"original_text": "uh",
|
||||
"corrected_text": "um",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "um"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "spoken_word",
|
||||
"id": 1,
|
||||
"original_text": "ha",
|
||||
"corrected_text": "ho",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "ho"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "glossary_1",
|
||||
"id": 1,
|
||||
"original_text": "gestures",
|
||||
"corrected_text": "Jesters",
|
||||
"confidence": 0.95
|
||||
"corrected_text": "Jesters"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -18,8 +18,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "foo bar",
|
||||
"corrected_text": "foo-bar",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "foo-bar"
|
||||
},
|
||||
{
|
||||
"proposal_index": 1,
|
||||
@@ -27,8 +26,7 @@
|
||||
"module_instance": "grammar_1",
|
||||
"id": 1,
|
||||
"original_text": "foo bar",
|
||||
"corrected_text": "foobar",
|
||||
"confidence": 0.9
|
||||
"corrected_text": "foobar"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -26,8 +26,7 @@
|
||||
"required": [
|
||||
"id",
|
||||
"original_text",
|
||||
"corrected_text",
|
||||
"confidence"
|
||||
"corrected_text"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
@@ -40,11 +39,6 @@
|
||||
},
|
||||
"corrected_text": {
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -13,8 +13,7 @@
|
||||
"required": [
|
||||
"id",
|
||||
"original_text",
|
||||
"corrected_text",
|
||||
"confidence"
|
||||
"corrected_text"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
@@ -27,11 +26,6 @@
|
||||
},
|
||||
"corrected_text": {
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
"required": [
|
||||
"correction_index",
|
||||
"approved",
|
||||
"confidence",
|
||||
"reason"
|
||||
],
|
||||
"properties": {
|
||||
@@ -24,11 +23,6 @@
|
||||
"approved": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"reason": {
|
||||
"type": "string"
|
||||
}
|
||||
|
||||
@@ -76,9 +76,6 @@ func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
|
||||
if !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("expected valid JSON schema for %q", schema.ID)
|
||||
}
|
||||
if !bytes.Contains(schema.JSONSchema, []byte("\n ")) {
|
||||
t.Fatalf("expected readable formatted JSON schema for %q", schema.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,14 +149,13 @@ func TestCorrectionRepairSchema(t *testing.T) {
|
||||
name, content string
|
||||
valid bool
|
||||
}{
|
||||
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0}}`, true},
|
||||
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
|
||||
{"withdraw", `{"proposal_ref":1,"action":"withdraw","correction":null}`, true},
|
||||
{"missing correction", `{"proposal_ref":1,"action":"withdraw"}`, false},
|
||||
{"extra field", `{"proposal_ref":1,"action":"withdraw","correction":null,"extra":1}`, false},
|
||||
{"zero reference", `{"proposal_ref":0,"action":"withdraw","correction":null}`, false},
|
||||
{"unknown action", `{"proposal_ref":1,"action":"other","correction":null}`, false},
|
||||
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x","confidence":1}}`, false},
|
||||
{"confidence outside range", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"x","corrected_text":"y","confidence":2}}`, false},
|
||||
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x"}}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var value any
|
||||
|
||||
@@ -105,7 +105,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
||||
return validators.Result{}, err
|
||||
}}
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
|
||||
}}
|
||||
sections := []chunking.Section{}
|
||||
for i := 0; i < 3; i++ {
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -20,15 +18,12 @@ func feedbackCandidates(input validateSectionCandidatesInput, validator contract
|
||||
}
|
||||
usableSpan := candidate.OriginalText != ""
|
||||
target := candidate.TargetSegmentID > 0
|
||||
confidence := !math.IsNaN(candidate.Confidence) && candidate.Confidence >= 0 && candidate.Confidence <= 1
|
||||
var applicable bool
|
||||
switch validator.Name() {
|
||||
case "proposal_shape":
|
||||
applicable = true
|
||||
case "no_effect":
|
||||
applicable = usableSpan
|
||||
case "confidence_threshold":
|
||||
applicable = confidence
|
||||
case "original_text_presence":
|
||||
applicable = usableSpan && target
|
||||
case "protected_terms":
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
)
|
||||
|
||||
func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) {
|
||||
candidate := func(index, target int, original, replacement string, confidence float64) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement, Confidence: confidence}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
candidate := func(index, target int, original, replacement string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
}
|
||||
var seen []int
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -29,7 +29,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog", .2), candidate(1, 0, "teh", "the", .2), candidate(2, 1, "teh", "the", .9), candidate(3, 1, "cat", "cat", .9)}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -41,7 +41,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
reasons[d.ProposalIndex][d.ReasonCode] = true
|
||||
}
|
||||
if !reasons[0][validators.ReasonLowConfidence] || !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
|
||||
if !reasons[0]["semantic_rejection"] || !reasons[1][validators.ReasonInvalidTargetSegment] || !reasons[2][string(proposals.SkipReasonAmbiguousOriginal)] || !reasons[3][validators.ReasonNoEffect] {
|
||||
t.Fatalf("findings = %+v", reasons)
|
||||
}
|
||||
if len(result.approved) != 0 || len(seen) != 1 || seen[0] != 0 {
|
||||
@@ -50,7 +50,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
|
||||
func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
for _, budget := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||
var later int
|
||||
@@ -82,7 +82,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
rejected := []ValidatorRejectedChange{{ValidatorName: "confidence_threshold", ProposalIndex: 2, ReasonCode: "low_confidence", Message: "confidence 0.2 below threshold 0.8"}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
|
||||
rejected := []ValidatorRejectedChange{{ValidatorName: "editorial_review", ProposalIndex: 2, ReasonCode: "llm_rejected", Message: "The proposed replacement is unsupported by the supplied context."}, {ValidatorName: "editorial_review", ProposalIndex: 2, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed", Message: "provider diagnostic secret"}, {ValidatorName: "other", ProposalIndex: 3, ReasonCode: "other"}}
|
||||
findings := repairFindings(2, rejected)
|
||||
if len(findings) != 2 || findings[0].Explanation != rejected[0].Message || findings[1].Status != "unavailable" || findings[1].Explanation != "assessment was unavailable" {
|
||||
t.Fatalf("findings: %+v", findings)
|
||||
@@ -91,8 +91,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
|
||||
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
@@ -102,7 +102,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
attempt := 1
|
||||
var paths []string
|
||||
for _, index := range []int{2, 3} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, DiagnosticsDir: dir, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -123,29 +123,12 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedbackUsesIndependentPreviewWhenConfidenceInvalid(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 2}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.approved) != 0 || len(result.decisions) != 2 || result.decisions[1].ValidatorName != "editorial_review" || !result.decisions[1].Approved {
|
||||
t.Fatalf("result: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
|
||||
identity := glossary.Identity{Collection: "party", ID: "jesters"}
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
return validators.ValidateProtectedTerms(req)
|
||||
}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
|
||||
@@ -37,7 +37,7 @@ func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest)
|
||||
if m.spec.Index == 2 {
|
||||
original, corrected = "Party Spell", "Party Spell!"
|
||||
}
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected, Confidence: 1}}}, nil
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected}}}, nil
|
||||
}
|
||||
|
||||
func TestOrderedWorkflowUsesIndependentSelectionsAndPriorEdits(t *testing.T) {
|
||||
|
||||
@@ -119,7 +119,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
findings := safeFindings(rejected)
|
||||
replay := &structuredoutput.Replay{Exchange: input.Exchange}
|
||||
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
|
||||
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText, Confidence: current.Confidence}, Findings: findings}).JSON()
|
||||
payload, err := (proposal_generation.RepairRequest{ProposalRef: position + 1, Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText}, Findings: findings}).JSON()
|
||||
if err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
@@ -198,7 +198,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
target = input.WorkingTranscript.Segments[localRef-1].ID
|
||||
}
|
||||
current = original
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText, Confidence: revision.Confidence}
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
|
||||
if debug {
|
||||
branch.Final = current
|
||||
}
|
||||
|
||||
@@ -46,12 +46,12 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
||||
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":"","confidence":0.9}}`}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -82,15 +82,15 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
calls int
|
||||
}{
|
||||
{"withdraw", []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}, nil, 1, "withdrawn", 1},
|
||||
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.1}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"dog","confidence":0.2}}`}, nil, 2, "exhausted", 2},
|
||||
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat","confidence":0.9}}`}, nil, 2, "rejected", 1},
|
||||
{"exhausted", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}`}, nil, 2, "exhausted", 2},
|
||||
{"no-effect revision", []string{`{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}`}, nil, 2, "rejected", 1},
|
||||
{"wrong reference", []string{`{"proposal_ref":2,"action":"withdraw","correction":null}`}, nil, 1, "unavailable", 1},
|
||||
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -114,15 +114,15 @@ type repairModule struct{ outcomeModule }
|
||||
func (m repairModule) RepairPromptID() string { return "repair" }
|
||||
|
||||
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl", Confidence: .1}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox","confidence":0.9}}`,
|
||||
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl","confidence":0.9}}`,
|
||||
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
|
||||
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.ConfidenceThresholdValidator{}}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -142,8 +142,8 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
|
||||
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -159,11 +159,11 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl","confidence":0.9}}`}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -185,8 +185,8 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -200,10 +200,10 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -237,8 +237,8 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warnings := 0
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{ConfidenceThreshold: .8, LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ConfidenceThresholdValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -267,7 +267,7 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
validation := &repairClient{secrets: []string{"validation-secret"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: .9}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
@@ -285,3 +285,24 @@ func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
t.Fatalf("credential in artifact: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
|
||||
func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
approved := req.RepairAttempt != nil
|
||||
for _, index := range preserved {
|
||||
if p.ProposalIndex == index {
|
||||
approved = true
|
||||
}
|
||||
}
|
||||
disposition, reason := validators.DispositionRejected, "llm_rejected"
|
||||
if approved {
|
||||
disposition, reason = validators.DispositionApproved, validators.ReasonApproved
|
||||
}
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: reason, Message: "The proposed replacement is unsupported by the supplied context."})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -13,10 +13,8 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -57,7 +55,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
|
||||
}}
|
||||
second := fakeModule{key: "second", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"first": failed, "second": second}})
|
||||
out, err := r.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "first", InstanceName: "first"}, {ModuleKey: "second", InstanceName: "second"}}})
|
||||
@@ -94,9 +92,9 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
|
||||
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true, RecoveryReasons: []outcomes.RecoveryReason{outcomes.RecoveryOutputRepaired}}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
|
||||
@@ -119,7 +117,7 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
|
||||
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
origin := errors.New("validator setup failed")
|
||||
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
|
||||
}}
|
||||
@@ -133,7 +131,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
input := RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "bad"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}}
|
||||
@@ -236,7 +234,7 @@ func TestRunnerOneModuleAppliesProposal(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"grammar": fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
|
||||
@@ -256,13 +254,13 @@ func TestRunnerModulesRunSequentially(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
if req.WorkingTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("second module did not see first module changes: %q", req.WorkingTranscript.Segments[0].Text)
|
||||
}
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}},
|
||||
}})
|
||||
|
||||
@@ -293,7 +291,7 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
|
||||
if req.Section.Index != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText, Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "a", CorrectedText: updatedText}}, nil
|
||||
}},
|
||||
"inspect": fakeModule{key: "inspect", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
ids := make([]int, len(req.WorkingTranscript.Segments))
|
||||
@@ -307,7 +305,7 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
|
||||
return nil, errors.New("later module did not see updated section text")
|
||||
}
|
||||
if req.Section.Index == 0 {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C"}}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}},
|
||||
@@ -449,7 +447,7 @@ func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *tes
|
||||
started <- seg.ID
|
||||
<-releaseByID[seg.ID]
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -584,7 +582,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
<-releaseSectionOne
|
||||
}
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text), Confidence: 1},
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
@@ -630,7 +628,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
|
||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -662,7 +660,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
|
||||
target := out.(*validators.LLMValidationResponse)
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -672,7 +670,7 @@ func TestRunnerSkippedRecorded(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
||||
@@ -688,7 +686,7 @@ func TestRunnerFailureReturnsPartialProgress(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh cat"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return nil, errors.New("boom")
|
||||
@@ -712,7 +710,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
|
||||
before := transcript.Segments[0].Text
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
|
||||
@@ -747,7 +745,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{allowAll},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
},
|
||||
},
|
||||
}})
|
||||
@@ -775,7 +773,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
||||
@@ -813,8 +811,8 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -831,14 +829,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}}
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
// Missing one decision triggers cardinality error.
|
||||
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
|
||||
}}
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
||||
|
||||
@@ -861,7 +859,7 @@ func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}})
|
||||
@@ -884,8 +882,8 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "b", CorrectedText: "", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: ""},
|
||||
{TargetSegmentID: 1, OriginalText: "b", CorrectedText: ""},
|
||||
},
|
||||
Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}},
|
||||
}, nil
|
||||
@@ -953,7 +951,7 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||
@@ -961,7 +959,7 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := runnerTestConfig()
|
||||
@@ -979,11 +977,11 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "reject"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := runnerTestConfig()
|
||||
@@ -1004,14 +1002,14 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "bad index"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := runnerTestConfig()
|
||||
@@ -1036,9 +1034,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
|
||||
|
||||
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
|
||||
{CorrectionIndex: 3, Approved: true, Confidence: .9, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "unexpected"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
|
||||
}}}}
|
||||
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
var laterIndices []int
|
||||
@@ -1057,9 +1055,9 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good", Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
{TargetSegmentID: 1, OriginalText: "bad", CorrectedText: "good"},
|
||||
}, Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}, Succeeded: true}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m": module}})
|
||||
@@ -1086,7 +1084,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := runnerTestConfig()
|
||||
@@ -1102,13 +1100,13 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: false, Confidence: 0.9, Reason: "dup"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
|
||||
}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
cfg := runnerTestConfig()
|
||||
@@ -1128,8 +1126,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
|
||||
|
||||
func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
llmValidator.SetTokenEstimator(lenEstimator{})
|
||||
@@ -1139,8 +1137,8 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40), Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40), Confidence: 1},
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1164,7 +1162,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
primarySecret := "runner-primary-secret"
|
||||
validationSecret := "runner-validation-secret"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
client.secrets = []string{primarySecret}
|
||||
client.preparedSecret = validationSecret
|
||||
@@ -1172,7 +1170,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 1}}, nil
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
@@ -1262,7 +1260,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1278,7 +1276,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
|
||||
ProposalDiagnosticsDir: diagDir,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1299,70 +1297,6 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
|
||||
client := &fakeProposalStructuredClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Thresholds.Grammar = 0.9
|
||||
factory := modules.NewFactory()
|
||||
out, err := New(factory).Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "hello ,world" {
|
||||
t.Fatalf("expected no changes due to confidence threshold, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 {
|
||||
t.Fatalf("expected validator rejection, got %+v", out.ModuleResults)
|
||||
}
|
||||
if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence {
|
||||
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
|
||||
client := &fakeProposalStructuredClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.5},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Thresholds.Glossary = 0.9
|
||||
factory := modules.NewFactory()
|
||||
out, err := New(factory).Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "There were gestures" {
|
||||
t.Fatalf("expected no changes due to glossary confidence threshold, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 {
|
||||
t.Fatalf("expected validator rejection, got %+v", out.ModuleResults)
|
||||
}
|
||||
if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence {
|
||||
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
origin := errors.New("original fatal failure")
|
||||
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
|
||||
@@ -2,7 +2,6 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
@@ -21,8 +20,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
|
||||
case c.OriginalText == "":
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
|
||||
case c.Confidence < 0.0 || c.Confidence > 1.0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
|
||||
default:
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
@@ -33,26 +30,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type ConfidenceThresholdValidator struct{}
|
||||
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
|
||||
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
threshold := req.Settings.ConfidenceThreshold
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if c.Confidence < threshold {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonLowConfidence, fmt.Sprintf("confidence %.4f below threshold %.4f", c.Confidence, threshold)))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type OriginalTextPresenceValidator struct{}
|
||||
|
||||
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }
|
||||
|
||||
@@ -11,7 +11,7 @@ func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) {
|
||||
switch reason {
|
||||
case ReasonInvalidTargetSegment, ReasonMissingTargetSegment,
|
||||
ReasonEmptyOriginalText, ReasonMissingOriginalText,
|
||||
ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText,
|
||||
ReasonNoEffect, ReasonEmptyResultingText,
|
||||
string(proposals.SkipReasonAmbiguousOriginal),
|
||||
string(proposals.SkipReasonInvalidProposal):
|
||||
return outcomes.ReasonCode(reason), true
|
||||
|
||||
@@ -5,14 +5,14 @@ import "testing"
|
||||
func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) {
|
||||
for _, reason := range []string{
|
||||
ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText,
|
||||
ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect,
|
||||
ReasonMissingOriginalText, ReasonNoEffect,
|
||||
ReasonEmptyResultingText, "ambiguous_original_text",
|
||||
} {
|
||||
if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason {
|
||||
t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok)
|
||||
}
|
||||
}
|
||||
for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
|
||||
for _, reason := range []string{ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
|
||||
if got, ok := DefectiveProposalReason(reason); ok {
|
||||
t.Fatalf("ordinary rejection %q classified as %q", reason, got)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ type validationTransport func(*http.Request) (*http.Response, error)
|
||||
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
|
||||
valid := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
responses []string
|
||||
|
||||
@@ -43,10 +43,9 @@ type LLMValidationRequest struct {
|
||||
|
||||
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
|
||||
type LLMValidationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Reason string `json:"reason"`
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type LLMValidationResponse struct {
|
||||
|
||||
@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
||||
nextClient := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
||||
nextValidator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
||||
}}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
|
||||
}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
||||
}}}
|
||||
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
||||
cause := errors.New("broken embedded schema")
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
|
||||
@@ -319,11 +319,6 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
|
||||
continue
|
||||
}
|
||||
d := entries[0]
|
||||
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
||||
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
|
||||
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
reasonCode := ReasonApproved
|
||||
if !d.Approved {
|
||||
reasonCode = "llm_rejected"
|
||||
|
||||
@@ -72,7 +72,7 @@ func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
|
||||
|
||||
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
}
|
||||
}
|
||||
@@ -151,8 +151,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: ""},
|
||||
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
|
||||
}}}}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
@@ -164,7 +164,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("validate err: %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
|
||||
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved || res.Decisions[0].Message != "" || res.Decisions[1].Message != "bad" {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(client.calls) != 1 {
|
||||
@@ -206,10 +206,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
approve := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
|
||||
}
|
||||
reject := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
|
||||
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
|
||||
}
|
||||
type expectedDecision struct {
|
||||
index int
|
||||
@@ -230,7 +230,6 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
||||
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
||||
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
||||
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
|
||||
@@ -245,7 +244,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
}
|
||||
for i, want := range tc.want {
|
||||
got := result.Decisions[i]
|
||||
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
|
||||
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision) {
|
||||
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
|
||||
}
|
||||
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
|
||||
@@ -289,7 +288,7 @@ func equalIndices(a, b []int) bool {
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
@@ -300,11 +299,11 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
@@ -335,7 +334,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
|
||||
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
@@ -374,7 +373,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.kind), func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -399,7 +398,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
const (
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyResultingText = "empty_resulting_segment"
|
||||
@@ -22,12 +21,10 @@ const (
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonInvalidTargetSegment = "invalid_target_segment_id"
|
||||
ReasonEmptyOriginalText = "empty_original_text"
|
||||
ReasonInvalidConfidence = "invalid_confidence"
|
||||
ReasonValidatorMalformed = "validator_response_malformed"
|
||||
ReasonValidatorInputTooLarge = "validator_input_too_large"
|
||||
ReasonMissingValidationDecision = "missing_validation_decision"
|
||||
ReasonDuplicateValidationDecision = "duplicate_validation_decision"
|
||||
ReasonInvalidValidationDecision = "invalid_validation_decision"
|
||||
)
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
|
||||
@@ -39,12 +39,11 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "Jesters",
|
||||
CorrectedText: "Gestures",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1},
|
||||
},
|
||||
},
|
||||
@@ -68,7 +67,6 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
|
||||
},
|
||||
@@ -99,7 +97,7 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
|
||||
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
|
||||
@@ -120,8 +118,7 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mira", Name: "Mira", Aliases: []string{"Captain Mira"}}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain", Confidence: 0.9,
|
||||
},
|
||||
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain"},
|
||||
}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
@@ -50,9 +48,6 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
|
||||
segment = §ion.segments[position]
|
||||
}
|
||||
correction := candidate.CorrectionProposal
|
||||
if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
|
||||
correction.Confidence = 0.5
|
||||
}
|
||||
preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy)
|
||||
if r.Previews != nil {
|
||||
r.Previews[candidate.ProposalIndex] = preview
|
||||
|
||||
@@ -5,21 +5,20 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
|
||||
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
|
||||
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
|
||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing decision") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
|
||||
t.Fatalf("expected unknown decision error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 1, "recieve", "receive", 0.7),
|
||||
}}
|
||||
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
|
||||
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
|
||||
}
|
||||
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
|
||||
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, "", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
mkCandidate(4, 1, " ", " ", 0.9),
|
||||
mkCandidate(0, 1, "teh", "the"),
|
||||
mkCandidate(1, 0, "teh", "the"),
|
||||
mkCandidate(2, 1, "", "the"),
|
||||
mkCandidate(4, 1, " ", " "),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
if !res.Decisions[4].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
|
||||
if !res.Decisions[3].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "missing", "x", 0.9),
|
||||
mkCandidate(2, 5, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "missing", "x"),
|
||||
mkCandidate(2, 5, "hello", "hi"),
|
||||
}}
|
||||
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
mkCandidate(2, 1, "hello ", "", 0.9),
|
||||
mkCandidate(3, 1, "hello world", "", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "hello world", " "),
|
||||
mkCandidate(2, 1, "hello ", ""),
|
||||
mkCandidate(3, 1, "hello world", ""),
|
||||
}}
|
||||
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
|
||||
func TestNoEffectValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hello", 0.9),
|
||||
mkCandidate(1, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hello"),
|
||||
mkCandidate(1, 1, "hello", "hi"),
|
||||
}}
|
||||
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
|
||||
mkCandidate(1, 1, "teh", "the", 0.9),
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I"),
|
||||
mkCandidate(1, 1, "teh", "the"),
|
||||
},
|
||||
}
|
||||
res, err := ValidateProtectedTerms(req)
|
||||
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
func TestStableReasonCodes(t *testing.T) {
|
||||
codes := []string{
|
||||
ReasonApproved,
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyResultingText,
|
||||
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
|
||||
@@ -50,8 +50,8 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
@@ -89,8 +89,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
terms := glossary.Vocabulary(protected, exclusions)
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1,
|
||||
}}
|
||||
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
policy := proposals.ReplacementPolicyRequireUnique
|
||||
decision, err := ValidateProtectedTerms(Request{ModuleKey: tc.module, WorkingTranscript: transcript, ReplacementPolicy: policy,
|
||||
ProtectedTerms: terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
|
||||
Reference in New Issue
Block a user