142 lines
6.2 KiB
Go
142 lines
6.2 KiB
Go
package llm
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
|
pk "go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
reply reply
|
|
again bool
|
|
reason outcomes.ReasonCode
|
|
}{
|
|
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}},
|
|
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, again: true},
|
|
{name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted},
|
|
{name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, tc.reply}, 0, 0, nil)
|
|
collector := NewObservationCollector()
|
|
client, err := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req := adapterRequest(responseschema.CorrectionSetKey)
|
|
req.CaptureExchange = true
|
|
base, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deliveries := 0
|
|
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
|
if turn.Response != nil {
|
|
deliveries++
|
|
if tc.reason != "" {
|
|
var failure *structuredoutput.Failure
|
|
if !errors.As(turn.Err, &failure) || failure.Reason != tc.reason {
|
|
t.Fatalf("failure=%v", turn.Err)
|
|
}
|
|
return nil, nil
|
|
}
|
|
if turn.Err != nil || len(turn.Response.Content) == 0 {
|
|
t.Fatalf("turn=%+v", turn)
|
|
}
|
|
if !tc.again {
|
|
return nil, nil
|
|
}
|
|
}
|
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
|
})
|
|
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
|
|
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
|
|
}
|
|
snapshot := collector.Snapshot()
|
|
summary := Summarize(snapshot)
|
|
if len(snapshot.Calls) != 2 || summary.ProviderAttempts != 2 || !summary.TerminalCoverageComplete {
|
|
t.Fatalf("summary=%+v calls=%+v", summary, snapshot.Calls)
|
|
}
|
|
for _, call := range snapshot.Calls {
|
|
if call.Finished.IsZero() {
|
|
t.Fatal("unfinished logical call")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
|
|
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}, 0, 0, nil)
|
|
collector := NewObservationCollector()
|
|
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
|
|
req := adapterRequest(responseschema.CorrectionSetKey)
|
|
req.CaptureExchange = true
|
|
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
hooks := collector.Hooks()
|
|
hooks.OnEvent = func(e pk.ExecutionEvent) {
|
|
if e.Kind == pk.ExecutionFinished {
|
|
cancel()
|
|
}
|
|
}
|
|
client, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, hooks)
|
|
deliveries := 0
|
|
_, err = client.Coordinate(ctx, base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
|
if turn.Response != nil {
|
|
deliveries++
|
|
if !errors.Is(turn.Err, context.Canceled) || turn.Response.Prepared == nil || turn.Response.FinalContent == "" || len(turn.Response.Content) != 0 {
|
|
t.Fatalf("canceled turn=%+v", turn)
|
|
}
|
|
return nil, turn.Err
|
|
}
|
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
|
})
|
|
if !errors.Is(err, context.Canceled) || deliveries != 1 {
|
|
t.Fatalf("deliveries=%d err=%v", deliveries, err)
|
|
}
|
|
if got := Summarize(collector.Snapshot()); got.ProviderAttempts != 2 || !got.TerminalCoverageComplete {
|
|
t.Fatalf("summary=%+v", got)
|
|
}
|
|
}
|
|
|
|
func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
|
|
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
|
|
collector := NewObservationCollector()
|
|
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
|
|
repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
|
|
validation, _ := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, collector.Hooks())
|
|
req := adapterRequest(responseschema.CorrectionSetKey)
|
|
req.CaptureExchange = true
|
|
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = repair.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
|
if turn.Response != nil {
|
|
if turn.Err != nil {
|
|
return nil, turn.Err
|
|
}
|
|
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
|
|
return nil, err
|
|
}
|
|
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
|
})
|
|
summary := Summarize(collector.Snapshot())
|
|
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
|
|
t.Fatalf("calls=%d summary=%+v err=%v", count.Load(), summary, err)
|
|
}
|
|
}
|