Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -53,26 +53,24 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
p.mu.Lock()
|
||||
p.requests = append(p.requests, req)
|
||||
p.mu.Unlock()
|
||||
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello","confidence":0.99},{"id":2,"original_text":"second","corrected_text":"Second","confidence":0.1},{"id":3,"original_text":"third","corrected_text":"Third","confidence":0.1}]}`
|
||||
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello"},{"id":2,"original_text":"second","corrected_text":"Second"},{"id":3,"original_text":"third","corrected_text":"Third"}]}`
|
||||
if validation {
|
||||
var indices []int
|
||||
var candidates []fixtureValidationItem
|
||||
for _, message := range req.Messages {
|
||||
if pos := strings.LastIndex(message.Content, "Corrections to validate:"); pos >= 0 {
|
||||
var items []struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[pos+len("Corrections to validate:"):])), &items); err != nil {
|
||||
http.Error(w, "bad validation", 400)
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
indices = append(indices, item.CorrectionIndex)
|
||||
}
|
||||
if items, ok := fixtureValidationItems(message.Content); ok {
|
||||
candidates = append(candidates, items...)
|
||||
}
|
||||
}
|
||||
var decisions []map[string]any
|
||||
for _, index := range indices {
|
||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": true, "confidence": 0.99, "reason": "controlled"})
|
||||
for _, item := range candidates {
|
||||
p.mu.Lock()
|
||||
approved := item.OriginalText != "third" && (item.OriginalText != "second" || p.attempts[2] > 1)
|
||||
p.mu.Unlock()
|
||||
reason := ""
|
||||
if !approved {
|
||||
reason = "The capitalization is unsupported in the supplied context."
|
||||
}
|
||||
decisions = append(decisions, map[string]any{"correction_index": item.CorrectionIndex, "approved": approved, "reason": reason})
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||
content = string(raw)
|
||||
@@ -99,15 +97,11 @@ func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
confidence := 0.1
|
||||
if attempt > 1 {
|
||||
confidence = 0.99
|
||||
}
|
||||
id := 2
|
||||
if invalid {
|
||||
id, confidence = 99, 0.99
|
||||
id = 99
|
||||
}
|
||||
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second","confidence":%v}}`, id, confidence)
|
||||
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second"}}`, id)
|
||||
}
|
||||
raw, _ := json.Marshal(content)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
@@ -121,7 +121,7 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
|
||||
return
|
||||
}
|
||||
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
|
||||
if p.mode == "invalid-credential-echo" {
|
||||
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
|
||||
@@ -131,45 +131,45 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
if proposalCall == 1 {
|
||||
content = `{"wrong":true}`
|
||||
} else {
|
||||
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X"},{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(p.mode, "span-") {
|
||||
switch p.mode {
|
||||
case "span-clean", "span-filter":
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"}]}`
|
||||
if p.mode == "span-filter" {
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"},{"id":3,"original_text":"uh","corrected_text":""}]}`
|
||||
}
|
||||
case "span-spacing":
|
||||
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" "}]}`
|
||||
case "span-negative":
|
||||
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":""}]}`
|
||||
}
|
||||
if validation {
|
||||
if p.mode == "span-clean" || p.mode == "span-filter" {
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"},{"correction_index":2,"approved":true,"reason":"ok"}]}`
|
||||
} else {
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||
}
|
||||
if p.mode == "span-negative" && validationCall == 2 {
|
||||
content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
|
||||
content = `{"validations":[{"correction_index":1,"approved":false,"reason":"would reverse meaning"}]}`
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(p.mode, "decision-") && !validation {
|
||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"}]}`
|
||||
if p.mode == "decision-mixed" {
|
||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
|
||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"},{"id":2,"original_text":"cat","corrected_text":"dog"},{"id":3,"original_text":"bad","corrected_text":"good"},{"id":4,"original_text":"slow","corrected_text":"fast"}]}`
|
||||
}
|
||||
}
|
||||
if validation && !strings.HasPrefix(p.mode, "span-") {
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||
if p.mode == "invalid-decision-after-repair" {
|
||||
if validationCall == 1 {
|
||||
content = `{"wrong":true}`
|
||||
} else {
|
||||
content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
content = `{"validations":[{"correction_index":9,"approved":true,"reason":"ok"}]}`
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -180,13 +180,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
continue
|
||||
}
|
||||
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
|
||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
|
||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "reason": "controlled review"})
|
||||
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
|
||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
|
||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "reason": "conflicting review"})
|
||||
}
|
||||
}
|
||||
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
|
||||
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
|
||||
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "reason": "unexpected review"})
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||
content = string(raw)
|
||||
@@ -196,26 +196,36 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||
}
|
||||
|
||||
// Read the supplied candidate JSON independently of the bundled instruction prose.
|
||||
type fixtureValidationItem struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
OriginalText string `json:"original_text"`
|
||||
}
|
||||
|
||||
func fixtureValidationItems(content string) ([]fixtureValidationItem, bool) {
|
||||
for offset := 0; offset < len(content); offset++ {
|
||||
if content[offset] != '[' {
|
||||
continue
|
||||
}
|
||||
var items []fixtureValidationItem
|
||||
if err := json.Unmarshal([]byte(content[offset:]), &items); err == nil && len(items) > 0 && items[0].CorrectionIndex > 0 {
|
||||
return items, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func fixtureValidationIndices(messages []struct {
|
||||
Content string `json:"content"`
|
||||
}) ([]int, error) {
|
||||
for _, message := range messages {
|
||||
const marker = "Corrections to validate:"
|
||||
position := strings.LastIndex(message.Content, marker)
|
||||
if position < 0 {
|
||||
continue
|
||||
if items, ok := fixtureValidationItems(message.Content); ok {
|
||||
indices := make([]int, len(items))
|
||||
for i, item := range items {
|
||||
indices[i] = item.CorrectionIndex
|
||||
}
|
||||
return indices, nil
|
||||
}
|
||||
var items []struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
|
||||
return nil, fmt.Errorf("decode validation request items: %w", err)
|
||||
}
|
||||
indices := make([]int, len(items))
|
||||
for i, item := range items {
|
||||
indices[i] = item.CorrectionIndex
|
||||
}
|
||||
return indices, nil
|
||||
}
|
||||
return nil, fmt.Errorf("validation request items not found")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user