Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -27,7 +27,7 @@ func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
|
||||
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
calls := 0
|
||||
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
|
||||
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present"}]}`}
|
||||
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
|
||||
output := outputs[len(outputs)-1]
|
||||
if calls < len(outputs) {
|
||||
@@ -76,16 +76,12 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
if calls != 2 || len(generated.Enriched) != 1 {
|
||||
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if low.Decisions[0].ReasonCode != validators.ReasonLowConfidence || missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
|
||||
t.Fatalf("domain checks or extra semantic retry: low=%+v missing=%+v calls=%d", low, missing, calls)
|
||||
if missing.Decisions[0].ReasonCode != validators.ReasonMissingOriginalText || calls != 2 {
|
||||
t.Fatalf("domain checks or extra semantic retry: missing=%+v calls=%d", missing, calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,10 +35,9 @@ type InteractionArtifacts struct {
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
SegmentReference int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
SegmentReference int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionSet is the reusable structured LLM response model for
|
||||
@@ -201,7 +200,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
TargetSegmentID: targetID,
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -121,7 +121,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
@@ -143,7 +143,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
diag := &captureDiagnosticsWriter{}
|
||||
@@ -175,7 +175,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
|
||||
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -190,7 +190,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
if len(result.Corrections) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 || result.Corrections[0].Confidence != 1.2 {
|
||||
if result.Corrections[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
@@ -238,8 +238,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
@@ -269,8 +269,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
@@ -316,7 +316,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
validationSecret := "proposal-validation-secret"
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
|
||||
},
|
||||
}
|
||||
t.Setenv("PRIMARY_SECRET", primarySecret)
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
client := &fakeStructuredClient{
|
||||
completion: structuredoutput.Response{Exchange: exchange},
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"}}},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 7}
|
||||
@@ -102,10 +102,10 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
|
||||
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
|
||||
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
|
||||
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: ""},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello"},
|
||||
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the"},
|
||||
}}}}
|
||||
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
ProposalRequest: contracts.ProposalRequest{
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
)
|
||||
|
||||
func TestRepairRequestJSONPreservesCurrentVersionAndFindings(t *testing.T) {
|
||||
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: "", Confidence: 0.7}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
|
||||
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: ""}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
|
||||
text, err := request.JSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestDecodeRepairResponse(t *testing.T) {
|
||||
name, content string
|
||||
valid bool
|
||||
}{
|
||||
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true},
|
||||
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, true},
|
||||
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
|
||||
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
|
||||
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},
|
||||
|
||||
Reference in New Issue
Block a user