Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -215,7 +215,7 @@ func TestCorrectionLedgerCombinesRepairLineage(t *testing.T) {
|
||||
revised.TargetSegmentID = 2
|
||||
revised.OriginalText = "uh "
|
||||
revised.CorrectedText = ""
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "confidence_threshold", ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "low_confidence"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{StageIndex: 0, ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusSuccess, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "original_text_presence", ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}, ValidatorRejected: []runner.ValidatorRejectedChange{{ProposalIndex: 3, ReasonCode: "llm_rejected"}}, AppliedChanges: []proposals.AppliedChange{{ProposalIndex: 3, TargetSegmentID: 2, OriginalText: "uh ", CorrectedText: ""}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 3, Original: original, Final: revised, Outcome: "applied", Attempts: []runner.RepairAttempt{{Revision: &revised, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 3, ValidatorName: "editorial_review", Approved: true, Disposition: validators.DispositionApproved, DiagnosticArtifactPath: "validation.json"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
if len(ledger) != 1 {
|
||||
t.Fatalf("ledger: %+v", ledger)
|
||||
@@ -233,7 +233,6 @@ func TestSkippedRepairLedgerKeepsApplicationReason(t *testing.T) {
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: instance},
|
||||
}
|
||||
revision := original
|
||||
revision.Confidence = 0.99
|
||||
skip := proposals.SkippedChange{ProposalIndex: 1, SkipReason: proposals.SkipReasonMissingOriginalText, Message: "source span no longer matches"}
|
||||
output := &runner.RunOutput{
|
||||
ModuleResults: []runner.ModuleResult{{ModuleInstance: instance, SkippedChanges: []proposals.SkippedChange{skip}}},
|
||||
@@ -259,7 +258,7 @@ func TestInterruptedRepairLedgerKeepsEarlierDecisions(t *testing.T) {
|
||||
original := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "grammar", ModuleInstance: "grammar_1"}}
|
||||
revision := original
|
||||
revision.CorrectedText = "fox"
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "confidence_threshold", ProposalIndex: 1, ReasonCode: "low_confidence", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
output := &runner.RunOutput{ModuleResults: []runner.ModuleResult{{ModuleKey: "grammar", ModuleInstance: "grammar_1", Status: runner.ModuleStatusFailed, ValidatorDecisions: []runner.ValidatorDecisionRecord{{ValidatorName: "original_text_presence", ProposalIndex: 1, ReasonCode: "missing_original_text", Disposition: validators.DispositionRejected}}}}, WorkUnits: []runner.WorkUnitOutcome{{ModuleInstance: "grammar_1", Repairs: []runner.RepairOutcome{{ProposalIndex: 1, Original: original, Final: revision, Outcome: "interrupted", Attempts: []runner.RepairAttempt{{Revision: &revision, Decisions: []runner.ValidatorDecisionRecord{{ProposalIndex: 1, ValidatorName: "editorial_review", Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}}}}}}}}, Candidates: []proposals.EnrichedCorrectionProposal{original}}
|
||||
ledger := BuildCorrectionLedger(CorrectionLedgerInput{RunOutput: output})
|
||||
var found *CorrectionLedgerEntry
|
||||
for i := range ledger {
|
||||
|
||||
@@ -29,7 +29,7 @@ func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.Transcr
|
||||
func (m partialModule) Propose(context.Context, contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result := contracts.ProposalResult{}
|
||||
for _, text := range []string{"Hello", "Greetings", "Hi"} {
|
||||
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text, Confidence: 1})
|
||||
result.Proposals = append(result.Proposals, proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: text})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Confidence: 1, Reason: "completed assessment"}}}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user