Give each correction repair branch its own outcome
This commit is contained in:
@@ -130,17 +130,9 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
if err != nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err})
|
||||
}
|
||||
var exchange *pk.Exchange
|
||||
if req.ContinueFrom != nil {
|
||||
var ok bool
|
||||
exchange, ok = req.ContinueFrom.State.(*pk.Exchange)
|
||||
if !ok || exchange == nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
|
||||
}
|
||||
}
|
||||
call.request = pk.RunRequest{
|
||||
PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables),
|
||||
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
|
||||
CaptureExchange: req.CaptureExchange,
|
||||
TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout,
|
||||
Validation: contract,
|
||||
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
|
||||
func TestRoleClientCoordinatesIndependentRepairConversations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
@@ -75,13 +75,21 @@ func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
|
||||
t.Fatalf("capture: %+v %v", initial, err)
|
||||
}
|
||||
vars["SectionJSON"] = "changed"
|
||||
exchange := initial.Exchange
|
||||
for _, finding := range []string{"first", "second"} {
|
||||
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, ContinueFrom: exchange}, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
turns := 0
|
||||
exhausted, err := repair.Coordinate(context.Background(), initial.Exchange, 2, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Err != nil {
|
||||
return nil, turn.Err
|
||||
}
|
||||
exchange = result.Exchange
|
||||
if turn.Response != nil {
|
||||
turns++
|
||||
}
|
||||
if turns == 2 {
|
||||
return nil, nil
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": []string{"first", "second"}[turns]}}, nil
|
||||
})
|
||||
if err != nil || exhausted || turns != 2 {
|
||||
t.Fatalf("coordinated dialogue: turns=%d exhausted=%v error=%v", turns, exhausted, err)
|
||||
}
|
||||
if !reflect.DeepEqual(models, []string{"proposal", "repair", "repair"}) {
|
||||
t.Fatalf("models = %v", models)
|
||||
@@ -99,18 +107,24 @@ func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
|
||||
!strings.Contains(string(first.Content), `"text":"first"`) || !reflect.DeepEqual(first, messages[2][len(prefix)+1]) {
|
||||
t.Fatalf("cached instruction changed during replay: first=%s replay=%s", first.Content, messages[2][len(prefix)+1].Content)
|
||||
}
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, ContinueFrom: initial.Exchange}, &map[string]any{})
|
||||
independent := func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil {
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}}, nil
|
||||
}
|
||||
_, err = repair.Coordinate(context.Background(), initial.Exchange, 1, independent)
|
||||
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
|
||||
t.Fatalf("independent branch: %v %+v", err, messages)
|
||||
}
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, ContinueFrom: &structuredoutput.Exchange{}}, &map[string]any{})
|
||||
_, err = repair.Coordinate(context.Background(), &structuredoutput.Exchange{}, 1, independent)
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || len(messages) != 4 {
|
||||
t.Fatalf("prefix failure: %v, calls %d", err, len(messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T) {
|
||||
func TestCoordinatedRepairUsesReplacementAndCountsStructuralRepair(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
@@ -152,11 +166,18 @@ func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, ContinueFrom: base.Exchange}, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
var result *structuredoutput.Response
|
||||
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil {
|
||||
result = turn.Response
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}}, nil
|
||||
})
|
||||
if err != nil || exhausted || result == nil {
|
||||
t.Fatalf("coordinated structural repair: result=%+v exhausted=%v error=%v", result, exhausted, err)
|
||||
}
|
||||
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
|
||||
if len(requests) != 3 || !strings.Contains(requests[1][len(requests[1])-1].Content, "Replace fixture") || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
|
||||
t.Fatalf("result=%+v calls=%d", result, len(requests))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package llm
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
@@ -17,6 +18,9 @@ func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exc
|
||||
if start != nil {
|
||||
exchange, _ = start.State.(*pk.Exchange)
|
||||
}
|
||||
if exchange == nil {
|
||||
return false, classifiedFailure(ctx, fmt.Errorf("%w: repair exchange is unavailable", pk.ErrInvalidRequest), nil)
|
||||
}
|
||||
var pending *pendingCall
|
||||
result, err := c.runtime.Engine.Coordinate(ctx, pk.CoordinateRequest{
|
||||
Start: exchange, AdditionalTurns: budget, Observer: c.hooks.OnEvent,
|
||||
|
||||
Reference in New Issue
Block a user