Remove confidence scoring and refine correction prompts

This commit is contained in:
2026-11-11 17:05:52 +00:00
parent bbf89bb4f5
commit d155bf4cb3
92 changed files with 448 additions and 786 deletions

View File

@@ -27,7 +27,7 @@ func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) {
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
calls := 0
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present"}]}`}
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
output := outputs[len(outputs)-1]
if calls < len(outputs) {
@@ -76,16 +76,12 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if calls != 2 || len(generated.Enriched) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
if missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
t.Fatalf("domain checks or extra semantic retry: missing=%+v calls=%d", missing, calls)
}
}

View File

@@ -35,10 +35,9 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
}
// StructuredCorrectionSet is the reusable structured LLM response model for
@@ -201,7 +200,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
TargetSegmentID: targetID,
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
}
corrections = append(corrections, candidate)

View File

@@ -86,7 +86,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
@@ -121,7 +121,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
req := defaultRequest(t)
@@ -143,7 +143,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
diag := &captureDiagnosticsWriter{}
@@ -175,7 +175,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
},
},
},
@@ -190,7 +190,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if len(result.Corrections) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
@@ -238,8 +238,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -269,8 +269,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
@@ -316,7 +316,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)

View File

@@ -22,7 +22,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
client := &fakeStructuredClient{
completion: structuredoutput.Response{Exchange: exchange},
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
},
}
section := contracts.SectionMetadata{Index: 7}
@@ -102,10 +102,10 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: ""},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello"},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the"},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{

View File

@@ -6,7 +6,7 @@ import (
)
func TestRepairRequestJSONPreservesCurrentVersionAndFindings(t *testing.T) {
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: "", Confidence: 0.7}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: ""}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
text, err := request.JSON()
if err != nil {
t.Fatal(err)

View File

@@ -13,7 +13,7 @@ func TestDecodeRepairResponse(t *testing.T) {
name, content string
valid bool
}{
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true},
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},