package llm import ( "context" "errors" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/prompts" pk "go.maximumdirect.net/promptkit" ) func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) { for _, tc := range []struct { name string reply reply again bool reason outcomes.ReasonCode }{ {name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}}, {name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, again: true}, {name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted}, {name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed}, } { t.Run(tc.name, func(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, tc.reply}, 0, 0, nil) collector := NewObservationCollector() client, err := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks()) if err != nil { t.Fatal(err) } req := adapterRequest(responseschema.CorrectionSetKey) req.CaptureExchange = true base, err := client.CompleteStructured(context.Background(), req, &map[string]any{}) if err != nil { t.Fatal(err) } deliveries := 0 exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) { if turn.Response != nil { deliveries++ if tc.reason != "" { var failure *structuredoutput.Failure if !errors.As(turn.Err, &failure) || failure.Reason != tc.reason { t.Fatalf("failure=%v", turn.Err) } return nil, nil } if turn.Err != nil || len(turn.Response.Content) == 0 { t.Fatalf("turn=%+v", turn) } if !tc.again { return nil, nil } } return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 { t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err) } snapshot := collector.Snapshot() summary := Summarize(snapshot) if len(snapshot.Calls) != 2 || summary.ProviderAttempts != 2 || !summary.TerminalCoverageComplete { t.Fatalf("summary=%+v calls=%+v", summary, snapshot.Calls) } for _, call := range snapshot.Calls { if call.Finished.IsZero() { t.Fatal("unfinished logical call") } } }) } } func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) { runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}}, 0, 0, nil) collector := NewObservationCollector() proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks()) req := adapterRequest(responseschema.CorrectionSetKey) req.CaptureExchange = true base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{}) if err != nil { t.Fatal(err) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() hooks := collector.Hooks() hooks.OnEvent = func(e pk.ExecutionEvent) { if e.Kind == pk.ExecutionFinished { cancel() } } client, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, hooks) deliveries := 0 _, err = client.Coordinate(ctx, base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) { if turn.Response != nil { deliveries++ if !errors.Is(turn.Err, context.Canceled) || turn.Response.Prepared == nil || turn.Response.FinalContent == "" || len(turn.Response.Content) != 0 { t.Fatalf("canceled turn=%+v", turn) } return nil, turn.Err } return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) if !errors.Is(err, context.Canceled) || deliveries != 1 { t.Fatalf("deliveries=%d err=%v", deliveries, err) } if got := Summarize(collector.Snapshot()); got.ProviderAttempts != 2 || !got.TerminalCoverageComplete { t.Fatalf("summary=%+v", got) } } func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) { runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, {200, `{"validations":[]}`}}, 0, 0, nil) collector := NewObservationCollector() proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks()) repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks()) validation, _ := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, collector.Hooks()) req := adapterRequest(responseschema.CorrectionSetKey) req.CaptureExchange = true base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{}) if err != nil { t.Fatal(err) } _, err = repair.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) { if turn.Response != nil { if turn.Err != nil { return nil, turn.Err } _, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{}) return nil, err } return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil }) summary := Summarize(collector.Snapshot()) if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete { t.Fatalf("calls=%d summary=%+v err=%v", count.Load(), summary, err) } }