package pipeline import ( "context" "encoding/json" "errors" "fmt" "reflect" "strings" "testing" "testing/synctest" "time" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return f(ctx, request) } func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine { t.Helper() settings := `profiles: - id: base endpoint: https://example.invalid/v1 model: initial-model api_key_required: false - id: correction-default endpoint: https://example.invalid/v1 model: correction-model api_key_required: false - id: chosen endpoint: https://example.invalid/v1 model: chosen-model api_key_required: false prompts: - id: initial default_profile: base inputs: [{name: source, required: true}] messages: - {role: system, content: 'Keep enduring constraints.'} - {role: user, content: 'Source: {{input "source"}}'} output: {format: json, validation_mode: json, repair_attempts: 0} - id: correction default_profile: correction-default inputs: [{name: guidance, required: true}] messages: - {role: user, content: 'Correct: {{input "guidance"}}'} output: {format: json, validation_mode: json, repair_attempts: 0} ` underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}} if len(extraInputs) > 0 { underlays = append(underlays, configkit.Underlay{Inputs: extraInputs}) } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: underlays, }}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client)) if err != nil { t.Fatal(err) } return engine } func nativeProducerInitial() promptkit.RunRequest { return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}} } func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) { var candidate struct { Value int `json:"value"` } if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil { return nil, nil, err } return &candidate.Value, nil, nil } func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) { return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil } func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) { return contracts.CandidateAssessment{}, nil } func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) { return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil } func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) { for _, test := range []struct { name string values []int turns int mode contracts.ExecutionMode wantAccepted bool }{ {"initial acceptance", []int{1}, 3, contracts.ModeStrict, true}, {"zero turns", []int{0}, 0, contracts.ModeStrict, false}, {"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true}, {"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true}, {"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false}, {"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false}, } { t.Run(test.name, func(t *testing.T) { calls, assessments, interpreted, fallbacks := 0, 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if calls >= len(test.values) { return nil, errors.New("unexpected extra provider call") } value := test.values[calls] calls++ return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessments++ if value == 1 { return acceptedProducerAssessment(ctx, value) } return rejectedProducerAssessment(ctx, value) }, } value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { interpreted++ return interpretProducerNumber(ctx, result) }, producerCorrection, func(_ context.Context, final *int) (int, error) { fallbacks++ if final == nil || *final != test.values[len(test.values)-1] { t.Fatalf("fallback did not receive the final usable candidate: %v", final) } return -1, nil }) if calls != len(test.values) || assessments != calls || interpreted != calls { t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values)) } switch { case test.wantAccepted: if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 { t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks) } case test.mode == contracts.ModePermissive: if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 { t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks) } default: if err == nil || outcome.Disposition != "" || fallbacks != 0 { t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks) } } }) } } func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) { for _, failCorrection := range []bool{false, true} { calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if failCorrection && calls == 1 { return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil } return &promptkit.GenerateResponse{Content: "not JSON"}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessments++ return rejectedProducerAssessment(ctx, value) }, } _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { interpretations++ return interpretProducerNumber(ctx, result) }, func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { corrections++ return producerCorrection(ctx, guidance) }, func(_ context.Context, candidate *int) (int, error) { fallbacks++ if candidate != nil { t.Fatal("structurally failed response recovered a prior candidate") } return -1, nil }) want := 0 if failCorrection { want = 1 } if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 { t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks) } } } func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) { failure := errors.New("provider unavailable") for _, failCorrection := range []bool{false, true} { calls, interpreted, assessed := 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if failCorrection && calls == 1 { return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil } return nil, failure })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessed++ return rejectedProducerAssessment(ctx, value) }, } _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { interpreted++ return interpretProducerNumber(ctx, result) }, producerCorrection, func(_ context.Context, candidate *int) (int, error) { if candidate != nil { t.Fatal("failed execution recovered an earlier candidate") } return -1, nil }) want := 0 if failCorrection { want = 1 } if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want { t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed) } } engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return nil, failure })) _, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil) if !errors.Is(err, failure) { t.Fatalf("native failure cause was lost: %v", err) } } func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) { calls, corrections := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) { return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil }, } _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { corrections++ return producerCorrection(ctx, guidance) }, func(context.Context, *int) (int, error) { return -1, nil }) if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 { t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections) } } func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) { for _, explicit := range []bool{false, true} { var requests []promptkit.GenerateRequest var profiles []string initialPrepared, correctionPrepared, observed := 0, 0, 0 initialObserved, correctionObserved := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { requests = append(requests, request) return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) initial := nativeProducerInitial() initial.SessionID = "source-session" if explicit { initial.ProfileID = "chosen" } initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ } initial.Observer = func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionFinished { initialObserved++ } } overrideModel := "override-model" operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel}, Observer: func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionFinished { observed++ } }, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { profiles = append(profiles, details.SelectedProfileID) }, }, } _, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { if !reflect.DeepEqual(guidance, []string{"Return value one."}) { t.Fatalf("unexpected correction guidance: %#v", guidance) } request, err := producerCorrection(ctx, guidance) request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ } request.Observer = func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionFinished { correctionObserved++ } } return request, err }, func(context.Context, *int) (int, error) { return -1, nil }) if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 { t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared) } if initialObserved != 1 || correctionObserved != 2 { t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved) } wantProfiles := []string{"base", "correction-default", "correction-default"} if explicit { wantProfiles = []string{"chosen", "chosen", "chosen"} } if !reflect.DeepEqual(profiles, wantProfiles) { t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles) } for i, request := range requests { if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID { t.Fatalf("selection lost on turn %d: %#v", i, request) } var content []string for _, message := range request.Prompt.Messages { content = append(content, message.Content) } joined := strings.Join(content, "\n") if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i { t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content) } } if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" { t.Fatal("helper mutated the caller's initial request") } } } func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) { for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} { t.Run(moment, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessed++ if moment == "inside assessment" { cancel() return acceptedProducerAssessment(ctx, value) } return rejectedProducerAssessment(ctx, value) }, Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionFinished { finished++ if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" { cancel() } } }}, } if moment == "before initial" { cancel() } _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { interpreted++ if moment == "inside interpretation" { cancel() } return interpretProducerNumber(ctx, result) }, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 { t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks) } switch moment { case "before initial": if calls != 0 || interpreted != 0 || assessed != 0 { t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed) } case "inside assessment": if calls != 1 || interpreted != 1 || assessed != 1 { t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed) } case "after initial execution": if calls != 1 || interpreted != 0 || assessed != 0 { t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed) } case "inside interpretation": if calls != 1 || interpreted != 1 || assessed != 0 { t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed) } default: if calls != 2 || interpreted != 1 || assessed != 1 { t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed) } } }) } } func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) { synctest.Test(t, func(t *testing.T) { for _, continuation := range []bool{false, true} { for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} { calls, fallbacks := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ if continuation && calls == 1 { return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil } <-ctx.Done() return nil, ctx.Err() })) initial := nativeProducerInitial() initial.OperationTimeout = time.Second parent := context.Background() _, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment}, initial, interpretProducerNumber, func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { request, err := producerCorrection(ctx, guidance) request.OperationTimeout = time.Second return request, err }, func(_ context.Context, final *int) (int, error) { fallbacks++ if final != nil { t.Fatal("timeout recovered an earlier candidate") } return -1, nil }) if parent.Err() != nil { t.Fatalf("native timeout canceled caller: %v", parent.Err()) } if mode == contracts.ModeStrict { if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 { t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks) } } else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 { t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks) } } } }) } func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) { for _, usable := range []bool{false, true} { calls, interpreted, assessed := 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessed++ return acceptedProducerAssessment(ctx, value) }, } value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) { interpreted++ candidate, _, err := interpretProducerNumber(ctx, result) if interpreted == 1 { if !usable { candidate = nil } return candidate, []string{"Use the supplied candidate handles."}, err } return candidate, nil, err }, func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) { if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) { t.Fatalf("module findings lost: %#v", guidance) } return producerCorrection(ctx, guidance) }, nil) wantAssessed := 1 if usable { wantAssessed = 2 } if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed { t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed) } } } func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) { failure := errors.New("assessment framework failure") fallbacks, calls := 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) { return contracts.CandidateAssessment{}, failure }} _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 { t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks) } }