Give each correction repair branch its own outcome

This commit is contained in:
2026-11-16 14:36:07 +00:00
parent a0c70e8dce
commit e2710d869e
11 changed files with 309 additions and 233 deletions

View File

@@ -130,17 +130,9 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
if err != nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err})
}
var exchange *pk.Exchange
if req.ContinueFrom != nil {
var ok bool
exchange, ok = req.ContinueFrom.State.(*pk.Exchange)
if !ok || exchange == nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
}
}
call.request = pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables),
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
CaptureExchange: req.CaptureExchange,
TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout,
Validation: contract,
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {

View File

@@ -21,7 +21,7 @@ import (
ck "go.maximumdirect.net/configkit"
)
func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
@@ -75,13 +75,21 @@ func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
t.Fatalf("capture: %+v %v", initial, err)
}
vars["SectionJSON"] = "changed"
exchange := initial.Exchange
for _, finding := range []string{"first", "second"} {
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, ContinueFrom: exchange}, &map[string]any{})
if err != nil {
t.Fatal(err)
turns := 0
exhausted, err := repair.Coordinate(context.Background(), initial.Exchange, 2, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Err != nil {
return nil, turn.Err
}
exchange = result.Exchange
if turn.Response != nil {
turns++
}
if turns == 2 {
return nil, nil
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
})
if err != nil || exhausted || turns != 2 {
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
}
if !reflect.DeepEqual(models, []string{"proposal", "repair", "repair"}) {
t.Fatalf("models = %v", models)
@@ -99,18 +107,24 @@ func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
!strings.Contains(string(first.Content), `"text":"first"`) || !reflect.DeepEqual(first, messages[2][len(prefix)+1]) {
t.Fatalf("cached instruction changed during replay: first=%s replay=%s", first.Content, messages[2][len(prefix)+1].Content)
}
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, ContinueFrom: initial.Exchange}, &map[string]any{})
independent := func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
}
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
t.Fatalf("independent branch: %v %+v", err, messages)
}
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, ContinueFrom: &structuredoutput.Exchange{}}, &map[string]any{})
_, err = repair.Coordinate(context.Background(), &structuredoutput.Exchange{}, 1, independent)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || len(messages) != 4 {
t.Fatalf("prefix failure: %v, calls %d", err, len(messages))
}
}
func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T) {
func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\n",
@@ -152,11 +166,18 @@ func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T
if err != nil {
t.Fatal(err)
}
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, ContinueFrom: base.Exchange}, &map[string]any{})
if err != nil {
t.Fatal(err)
var result *structuredoutput.Response
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
result = turn.Response
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
})
if err != nil || exhausted || result == nil {
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)
}
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
if len(requests) != 3 || !strings.Contains(requests[1][len(requests[1])-1].Content, "Replace fixture") || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
t.Fatalf("result=%+v calls=%d", result, len(requests))
}
}

View File

@@ -3,6 +3,7 @@ package llm
import (
"context"
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
@@ -17,6 +18,9 @@ func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exc
if start != nil {
exchange, _ = start.State.(*pk.Exchange)
}
if exchange == nil {
return false, classifiedFailure(ctx, fmt.Errorf("%w: repair exchange is unavailable", pk.ErrInvalidRequest), nil)
}
var pending *pendingCall
result, err := c.runtime.Engine.Coordinate(ctx, pk.CoordinateRequest{
Start: exchange, AdditionalTurns: budget, Observer: c.hooks.OnEvent,