Files
notarius/internal/framework/pipeline/producer_test.go

523 lines
23 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"testing/synctest"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type producerClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (f producerClientFunc) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return f(ctx, request)
}
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine {
t.Helper()
settings := `profiles:
- id: base
endpoint: https://example.invalid/v1
model: initial-model
api_key_required: false
- id: correction-default
endpoint: https://example.invalid/v1
model: correction-model
api_key_required: false
- id: chosen
endpoint: https://example.invalid/v1
model: chosen-model
api_key_required: false
prompts:
- id: initial
default_profile: base
inputs: [{name: source, required: true}]
messages:
- {role: system, content: 'Keep enduring constraints.'}
- {role: user, content: 'Source: {{input "source"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
- id: correction
default_profile: correction-default
inputs: [{name: guidance, required: true}]
messages:
- {role: user, content: 'Correct: {{input "guidance"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}
if len(extraInputs) > 0 {
underlays = append(underlays, configkit.Underlay{Inputs: extraInputs})
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: underlays,
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
}
func nativeProducerInitial() promptkit.RunRequest {
return promptkit.RunRequest{PromptID: "initial", Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline("fixed source")}}
}
func interpretProducerNumber(_ context.Context, result *promptkit.RunResult) (*int, []string, error) {
var candidate struct {
Value int `json:"value"`
}
if err := json.Unmarshal([]byte(result.RawOutput), &candidate); err != nil {
return nil, nil, err
}
return &candidate.Value, nil, nil
}
func producerCorrection(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n"))}}, nil
}
func acceptedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, nil
}
func rejectedProducerAssessment(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "domain", Decision: contracts.AssessmentRejected, Result: contracts.ValidationResult{ReasonCode: "not_one", Message: "operator diagnostic", CorrectionGuidance: "Return value one."}}}}, nil
}
func TestNativeProducerAcceptsOnlyAssessedCandidates(t *testing.T) {
for _, test := range []struct {
name string
values []int
turns int
mode contracts.ExecutionMode
wantAccepted bool
}{
{"initial acceptance", []int{1}, 3, contracts.ModeStrict, true},
{"zero turns", []int{0}, 0, contracts.ModeStrict, false},
{"correction acceptance", []int{0, 1}, 3, contracts.ModeStrict, true},
{"final turn acceptance", []int{0, 0, 1}, 2, contracts.ModeStrict, true},
{"exhaustion", []int{0, 0, 0}, 2, contracts.ModeStrict, false},
{"permissive exhaustion", []int{0, 0, 0}, 2, contracts.ModePermissive, false},
} {
t.Run(test.name, func(t *testing.T) {
calls, assessments, interpreted, fallbacks := 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if calls >= len(test.values) {
return nil, errors.New("unexpected extra provider call")
}
value := test.values[calls]
calls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: test.mode, MaxAdditionalTurns: test.turns}, ReuseEligible: true,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
if value == 1 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(_ context.Context, final *int) (int, error) {
fallbacks++
if final == nil || *final != test.values[len(test.values)-1] {
t.Fatalf("fallback did not receive the final usable candidate: %v", final)
}
return -1, nil
})
if calls != len(test.values) || assessments != calls || interpreted != calls {
t.Fatalf("provider=%d interpretation=%d assessment=%d; want %d each", calls, interpreted, assessments, len(test.values))
}
switch {
case test.wantAccepted:
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || !outcome.ReuseEligible || fallbacks != 0 {
t.Fatalf("accepted = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
}
case test.mode == contracts.ModePermissive:
if err != nil || value != -1 || outcome.Disposition != contracts.ProducerFallback || outcome.ReuseEligible || fallbacks != 1 {
t.Fatalf("fallback = %d %#v %v, calls %d", value, outcome, err, fallbacks)
}
default:
if err == nil || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("strict rejection = %d %#v %v, fallback calls %d", value, outcome, err, fallbacks)
}
}
})
}
}
func TestNativeProducerDoesNotInterpretFailedNativeValidation(t *testing.T) {
for _, failCorrection := range []bool{false, true} {
calls, interpretations, assessments, corrections, fallbacks := 0, 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if failCorrection && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpretations++
return interpretProducerNumber(ctx, result)
},
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
corrections++
return producerCorrection(ctx, guidance)
},
func(_ context.Context, candidate *int) (int, error) {
fallbacks++
if candidate != nil {
t.Fatal("structurally failed response recovered a prior candidate")
}
return -1, nil
})
want := 0
if failCorrection {
want = 1
}
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpretations != want || assessments != want || corrections != want || fallbacks != 1 {
t.Fatalf("failed native status: %#v %v, calls %d/%d/%d/%d/%d", outcome, err, calls, interpretations, assessments, corrections, fallbacks)
}
}
}
func TestNativeProducerFailureDoesNotRecoverEarlierCandidate(t *testing.T) {
failure := errors.New("provider unavailable")
for _, failCorrection := range []bool{false, true} {
calls, interpreted, assessed := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if failCorrection && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
return nil, failure
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(_ context.Context, candidate *int) (int, error) {
if candidate != nil {
t.Fatal("failed execution recovered an earlier candidate")
}
return -1, nil
})
want := 0
if failCorrection {
want = 1
}
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != want+1 || interpreted != want || assessed != want {
t.Fatalf("provider failure = %#v %v, calls %d/%d/%d", outcome, err, calls, interpreted, assessed)
}
}
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, failure
}))
_, _, err := RunProducer(context.Background(), engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment}, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if !errors.Is(err, failure) {
t.Fatalf("native failure cause was lost: %v", err)
}
}
func TestNativeProducerUnavailableAssessmentSuppliesNoCorrection(t *testing.T) {
calls, corrections := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5},
Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{Decision: contracts.AssessmentUnavailable, Failure: errors.New("provider unavailable")}}}, nil
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
corrections++
return producerCorrection(ctx, guidance)
},
func(context.Context, *int) (int, error) { return -1, nil })
if err != nil || outcome.Disposition != contracts.ProducerFallback || calls != 1 || corrections != 0 {
t.Fatalf("unavailable assessment = %#v %v, calls %d/%d", outcome, err, calls, corrections)
}
}
func TestNativeProducerPreservesConversationAndExplicitSelections(t *testing.T) {
for _, explicit := range []bool{false, true} {
var requests []promptkit.GenerateRequest
var profiles []string
initialPrepared, correctionPrepared, observed := 0, 0, 0
initialObserved, correctionObserved := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
requests = append(requests, request)
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
initial := nativeProducerInitial()
initial.SessionID = "source-session"
if explicit {
initial.ProfileID = "chosen"
}
initial.OnPrepared = func(context.Context, promptkit.PreparedRun) { initialPrepared++ }
initial.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
initialObserved++
}
}
overrideModel := "override-model"
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment,
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: overrideModel},
Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
observed++
}
},
OnPrepared: func(_ context.Context, details promptkit.PreparedRun) {
profiles = append(profiles, details.SelectedProfileID)
},
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
if !reflect.DeepEqual(guidance, []string{"Return value one."}) {
t.Fatalf("unexpected correction guidance: %#v", guidance)
}
request, err := producerCorrection(ctx, guidance)
request.OnPrepared = func(context.Context, promptkit.PreparedRun) { correctionPrepared++ }
request.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
correctionObserved++
}
}
return request, err
}, func(context.Context, *int) (int, error) { return -1, nil })
if err != nil || outcome.Disposition != contracts.ProducerFallback || len(requests) != 3 || observed != 3 || initialPrepared != 1 || correctionPrepared != 2 {
t.Fatalf("native continuation = %#v %v, counts %d/%d/%d/%d", outcome, err, len(requests), observed, initialPrepared, correctionPrepared)
}
if initialObserved != 1 || correctionObserved != 2 {
t.Fatalf("existing request observers were lost or duplicated: %d/%d", initialObserved, correctionObserved)
}
wantProfiles := []string{"base", "correction-default", "correction-default"}
if explicit {
wantProfiles = []string{"chosen", "chosen", "chosen"}
}
if !reflect.DeepEqual(profiles, wantProfiles) {
t.Fatalf("profiles = %#v, want %#v", profiles, wantProfiles)
}
for i, request := range requests {
if request.Target.Model != overrideModel || request.Prompt.SessionID != initial.SessionID {
t.Fatalf("selection lost on turn %d: %#v", i, request)
}
var content []string
for _, message := range request.Prompt.Messages {
content = append(content, message.Content)
}
joined := strings.Join(content, "\n")
if strings.Count(joined, "fixed source") != 1 || strings.Count(joined, "Keep enduring constraints.") != 1 || len(request.Prompt.Messages) != 2+2*i {
t.Fatalf("conversation replayed or lost history on turn %d: %#v", i, content)
}
}
if initial.CaptureExchange || initial.Execution != nil || initial.ProfileID == "base" {
t.Fatal("helper mutated the caller's initial request")
}
}
}
func TestNativeProducerCancellationDoesNotDeliverOrFallback(t *testing.T) {
for _, moment := range []string{"before initial", "after initial execution", "inside interpretation", "inside assessment", "before continuation delivery"} {
t.Run(moment, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls, interpreted, assessed, finished, fallbacks := 0, 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
if moment == "inside assessment" {
cancel()
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
Execution: contracts.ExecutionOptions{Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
finished++
if (moment == "before continuation delivery" && finished == 2) || moment == "after initial execution" {
cancel()
}
}
}},
}
if moment == "before initial" {
cancel()
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
if moment == "inside interpretation" {
cancel()
}
return interpretProducerNumber(ctx, result)
},
producerCorrection,
func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("cancellation = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
switch moment {
case "before initial":
if calls != 0 || interpreted != 0 || assessed != 0 {
t.Fatalf("work started after cancellation: %d/%d/%d", calls, interpreted, assessed)
}
case "inside assessment":
if calls != 1 || interpreted != 1 || assessed != 1 {
t.Fatalf("canceled assessment repeated: %d/%d/%d", calls, interpreted, assessed)
}
case "after initial execution":
if calls != 1 || interpreted != 0 || assessed != 0 {
t.Fatalf("canceled native result was interpreted: %d/%d/%d", calls, interpreted, assessed)
}
case "inside interpretation":
if calls != 1 || interpreted != 1 || assessed != 0 {
t.Fatalf("assessment started after cancellation: %d/%d/%d", calls, interpreted, assessed)
}
default:
if calls != 2 || interpreted != 1 || assessed != 1 {
t.Fatalf("undelivered response assessed: %d/%d/%d", calls, interpreted, assessed)
}
}
})
}
}
func TestNativeProducerOperationTimeoutKeepsCallerActive(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
for _, continuation := range []bool{false, true} {
for _, mode := range []contracts.ExecutionMode{contracts.ModeStrict, contracts.ModePermissive} {
calls, fallbacks := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if continuation && calls == 1 {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}
<-ctx.Done()
return nil, ctx.Err()
}))
initial := nativeProducerInitial()
initial.OperationTimeout = time.Second
parent := context.Background()
_, outcome, err := RunProducer(parent, engine, contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: mode, MaxAdditionalTurns: 2}, Assess: rejectedProducerAssessment},
initial, interpretProducerNumber,
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
request, err := producerCorrection(ctx, guidance)
request.OperationTimeout = time.Second
return request, err
},
func(_ context.Context, final *int) (int, error) {
fallbacks++
if final != nil {
t.Fatal("timeout recovered an earlier candidate")
}
return -1, nil
})
if parent.Err() != nil {
t.Fatalf("native timeout canceled caller: %v", parent.Err())
}
if mode == contracts.ModeStrict {
if !errors.Is(err, promptkit.ErrOperationTimeout) || !errors.Is(err, context.DeadlineExceeded) || outcome.Disposition != "" || fallbacks != 0 {
t.Fatalf("strict timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
} else if err != nil || outcome.Disposition != contracts.ProducerFallback || fallbacks != 1 {
t.Fatalf("permissive timeout = %#v %v, fallback calls %d", outcome, err, fallbacks)
}
}
}
})
}
func TestNativeProducerInterpretationFindingsUseNativeCorrection(t *testing.T) {
for _, usable := range []bool{false, true} {
calls, interpreted, assessed := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, ReuseEligible: true,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return acceptedProducerAssessment(ctx, value)
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(),
func(ctx context.Context, result *promptkit.RunResult) (*int, []string, error) {
interpreted++
candidate, _, err := interpretProducerNumber(ctx, result)
if interpreted == 1 {
if !usable {
candidate = nil
}
return candidate, []string{"Use the supplied candidate handles."}, err
}
return candidate, nil, err
},
func(ctx context.Context, guidance []string) (promptkit.RunRequest, error) {
if !reflect.DeepEqual(guidance, []string{"Use the supplied candidate handles."}) {
t.Fatalf("module findings lost: %#v", guidance)
}
return producerCorrection(ctx, guidance)
}, nil)
wantAssessed := 1
if usable {
wantAssessed = 2
}
if err != nil || outcome.Disposition != contracts.ProducerAccepted || value != 1 || calls != 2 || interpreted != 2 || assessed != wantAssessed {
t.Fatalf("interpretation correction = %d %#v %v, calls %d/%d/%d", value, outcome, err, calls, interpreted, assessed)
}
}
}
func TestNativeProducerAssessmentContractFailureDoesNotFallback(t *testing.T) {
failure := errors.New("assessment framework failure")
fallbacks, calls := 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Assess: func(context.Context, int) (contracts.CandidateAssessment, error) {
return contracts.CandidateAssessment{}, failure
}}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, failure) || outcome.Disposition != "" || calls != 1 || fallbacks != 0 {
t.Fatalf("framework failure = %#v %v, calls %d/%d", outcome, err, calls, fallbacks)
}
}