Remove confidence scoring and refine correction prompts

This commit is contained in:
2026-11-11 17:05:52 +00:00
parent bbf89bb4f5
commit d155bf4cb3
92 changed files with 448 additions and 786 deletions

View File

@@ -53,26 +53,24 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
p.mu.Lock()
p.requests = append(p.requests, req)
p.mu.Unlock()
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello","confidence":0.99},{"id":2,"original_text":"second","corrected_text":"Second","confidence":0.1},{"id":3,"original_text":"third","corrected_text":"Third","confidence":0.1}]}`
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello"},{"id":2,"original_text":"second","corrected_text":"Second"},{"id":3,"original_text":"third","corrected_text":"Third"}]}`
if validation {
var indices []int
var candidates []fixtureValidationItem
for _, message := range req.Messages {
if pos := strings.LastIndex(message.Content, "Corrections to validate:"); pos >= 0 {
var items []struct {
CorrectionIndex int `json:"correction_index"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[pos+len("Corrections to validate:"):])), &items); err != nil {
http.Error(w, "bad validation", 400)
return
}
for _, item := range items {
indices = append(indices, item.CorrectionIndex)
}
if items, ok := fixtureValidationItems(message.Content); ok {
candidates = append(candidates, items...)
}
}
var decisions []map[string]any
for _, index := range indices {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": true, "confidence": 0.99, "reason": "controlled"})
for _, item := range candidates {
p.mu.Lock()
approved := item.OriginalText != "third" && (item.OriginalText != "second" || p.attempts[2] > 1)
p.mu.Unlock()
reason := ""
if !approved {
reason = "The capitalization is unsupported in the supplied context."
}
decisions = append(decisions, map[string]any{"correction_index": item.CorrectionIndex, "approved": approved, "reason": reason})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
@@ -99,15 +97,11 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
return
}
confidence := 0.1
if attempt > 1 {
confidence = 0.99
}
id := 2
if invalid {
id, confidence = 99, 0.99
id = 99
}
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second","confidence":%v}}`, id, confidence)
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second"}}`, id)
}
raw, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")

View File

@@ -121,7 +121,7 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
return
}
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
if p.mode == "invalid-credential-echo" {
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
@@ -131,45 +131,45 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
if proposalCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X"},{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
}
}
if strings.HasPrefix(p.mode, "span-") {
switch p.mode {
case "span-clean", "span-filter":
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"}]}`
if p.mode == "span-filter" {
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"},{"id":3,"original_text":"uh","corrected_text":""}]}`
}
case "span-spacing":
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" "}]}`
case "span-negative":
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":""}]}`
}
if validation {
if p.mode == "span-clean" || p.mode == "span-filter" {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"},{"correction_index":2,"approved":true,"reason":"ok"}]}`
} else {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
}
if p.mode == "span-negative" && validationCall == 2 {
content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
content = `{"validations":[{"correction_index":1,"approved":false,"reason":"would reverse meaning"}]}`
}
}
}
if strings.HasPrefix(p.mode, "decision-") && !validation {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"}]}`
if p.mode == "decision-mixed" {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"},{"id":2,"original_text":"cat","corrected_text":"dog"},{"id":3,"original_text":"bad","corrected_text":"good"},{"id":4,"original_text":"slow","corrected_text":"fast"}]}`
}
}
if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
if p.mode == "invalid-decision-after-repair" {
if validationCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}`
content = `{"validations":[{"correction_index":9,"approved":true,"reason":"ok"}]}`
}
}
}
@@ -180,13 +180,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
continue
}
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "reason": "conflicting review"})
}
}
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "reason": "unexpected review"})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
@@ -196,26 +196,36 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
// Read the supplied candidate JSON independently of the bundled instruction prose.
type fixtureValidationItem struct {
CorrectionIndex int `json:"correction_index"`
OriginalText string `json:"original_text"`
}
func fixtureValidationItems(content string) ([]fixtureValidationItem, bool) {
for offset := 0; offset < len(content); offset++ {
if content[offset] != '[' {
continue
}
var items []fixtureValidationItem
if err := json.Unmarshal([]byte(content[offset:]), &items); err == nil && len(items) > 0 && items[0].CorrectionIndex > 0 {
return items, true
}
}
return nil, false
}
func fixtureValidationIndices(messages []struct {
Content string `json:"content"`
}) ([]int, error) {
for _, message := range messages {
const marker = "Corrections to validate:"
position := strings.LastIndex(message.Content, marker)
if position < 0 {
continue
if items, ok := fixtureValidationItems(message.Content); ok {
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices, nil
}
var items []struct {
CorrectionIndex int `json:"correction_index"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
return nil, fmt.Errorf("decode validation request items: %w", err)
}
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices, nil
}
return nil, fmt.Errorf("validation request items not found")
}