Files
audita/internal/framework/llm/coordinate_test.go

142 lines
6.2 KiB
Go
Raw Normal View History

package llm
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "go.maximumdirect.net/promptkit"
)
func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
for _, tc := range []struct {
name string
reply reply
again bool
reason outcomes.ReasonCode
}{
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}},
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, again: true},
{name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted},
{name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed},
} {
t.Run(tc.name, func(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, tc.reply}, 0, 0, nil)
collector := NewObservationCollector()
client, err := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
if err != nil {
t.Fatal(err)
}
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
deliveries := 0
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
deliveries++
if tc.reason != "" {
var failure *structuredoutput.Failure
if !errors.As(turn.Err, &failure) || failure.Reason != tc.reason {
t.Fatalf("failure=%v", turn.Err)
}
return nil, nil
}
if turn.Err != nil || len(turn.Response.Content) == 0 {
t.Fatalf("turn=%+v", turn)
}
if !tc.again {
return nil, nil
}
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
}
snapshot := collector.Snapshot()
summary := Summarize(snapshot)
if len(snapshot.Calls) != 2 || summary.ProviderAttempts != 2 || !summary.TerminalCoverageComplete {
t.Fatalf("summary=%+v calls=%+v", summary, snapshot.Calls)
}
for _, call := range snapshot.Calls {
if call.Finished.IsZero() {
t.Fatal("unfinished logical call")
}
}
})
}
}
func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
hooks := collector.Hooks()
hooks.OnEvent = func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionFinished {
cancel()
}
}
client, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, hooks)
deliveries := 0
_, err = client.Coordinate(ctx, base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
deliveries++
if !errors.Is(turn.Err, context.Canceled) || turn.Response.Prepared == nil || turn.Response.FinalContent == "" || len(turn.Response.Content) != 0 {
t.Fatalf("canceled turn=%+v", turn)
}
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if !errors.Is(err, context.Canceled) || deliveries != 1 {
t.Fatalf("deliveries=%d err=%v", deliveries, err)
}
if got := Summarize(collector.Snapshot()); got.ProviderAttempts != 2 || !got.TerminalCoverageComplete {
t.Fatalf("summary=%+v", got)
}
}
func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
validation, _ := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, collector.Hooks())
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
_, err = repair.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
if turn.Err != nil {
return nil, turn.Err
}
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
return nil, err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, CaptureDiagnostics: true, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
summary := Summarize(collector.Snapshot())
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
t.Fatalf("calls=%d summary=%+v err=%v", count.Load(), summary, err)
}
}