package pipeline import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) type producerCacheStore struct { inner contracts.ArtifactCache reads, writes int readErr, writeErr error beforeWrite func(string, []byte) } func (store *producerCacheStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) { store.reads++ if store.readErr != nil { return nil, store.readErr } return store.inner.Lookup(ctx, mode, hash) } func (store *producerCacheStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error { store.writes++ if store.beforeWrite != nil { store.beforeWrite(hash, content) } if store.writeErr != nil { return store.writeErr } return store.inner.Store(ctx, mode, hash, content) } func numberCacheOperation(store contracts.ArtifactCache) contracts.ProducerOperation[int] { return contracts.ProducerOperation[int]{ Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheAuto, MaxAdditionalTurns: 1}, ReuseEligible: true, Assess: acceptedProducerAssessment, Cache: &contracts.ProducerCache[int]{ Store: store, Encode: func(value int) ([]byte, error) { return json.Marshal(value) }, Decode: func(content []byte) (int, error) { var value int err := json.Unmarshal(content, &value) return value, err }, }, } } func filesystemProducerCache(t *testing.T) (string, *producerCacheStore) { t.Helper() root := t.TempDir() store, err := artifactcache.NewFilesystemStore(root) if err != nil { t.Fatal(err) } return root, &producerCacheStore{inner: store} } func TestProducerCachesAppliedCorrectionUnderInitialKeyAndReusesWithoutExecution(t *testing.T) { root, store := filesystemProducerCache(t) calls, assessments, nativePrepared, events := 0, 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { value := 2 if calls == 0 { value = 0 } calls++ return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil })) operation := numberCacheOperation(store) operation.Execution = contracts.ExecutionOptions{ OnPrepared: func(context.Context, promptkit.PreparedRun) { nativePrepared++ }, Observer: func(promptkit.ExecutionEvent) { events++ }, } var keys []string var decisions []string operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error { decisions = append(decisions, observation.Decision) if observation.Prepared != nil { if observation.Prepared.MessageContentHash == "" || len(observation.Prepared.Messages) == 0 { t.Fatal("lost prepared details") } keys = append(keys, observation.MessageContentHash) } return nil } operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessments++ if value == 20 { return acceptedProducerAssessment(ctx, value) } return rejectedProducerAssessment(ctx, value) } // The response is a proposal; the cache must contain the applied typed value, // not the private response or an earlier candidate. interpretApplied := func(ctx context.Context, response *promptkit.RunResult) (*int, []string, error) { value, guidance, err := interpretProducerNumber(ctx, response) if value != nil { *value *= 10 } return value, guidance, err } store.beforeWrite = func(hash string, content []byte) { if assessments != 2 || calls != 2 || string(content) != "20" || hash != keys[0] { t.Fatalf("early or incorrect cache store: %s %q, calls=%d assessments=%d", hash, content, calls, assessments) } } value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretApplied, producerCorrection, nil) if err != nil || value != 20 || outcome.Disposition != contracts.ProducerAccepted || store.writes != 1 || nativePrepared != 2 || events == 0 { t.Fatalf("accepted correction = %d %#v %v; writes %d, prepared %d, events %d", value, outcome, err, store.writes, nativePrepared, events) } content, err := os.ReadFile(filepath.Join(root, keys[0]+".json")) if err != nil || string(content) != "20" { t.Fatalf("stored artifact = %q, %v", content, err) } priorEvents := events store.beforeWrite = nil initial := nativeProducerInitial() initial.SessionID, initial.ProfileID = "another-session", "chosen" operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"} operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) { t.Fatal("cache hit reassessed") return contracts.CandidateAssessment{}, nil } value, outcome, err = RunProducer(context.Background(), engine, operation, initial, func(context.Context, *promptkit.RunResult) (*int, []string, error) { t.Fatal("cache hit interpreted a model result") return nil, nil, nil }, producerCorrection, nil) if err != nil || value != 20 || !outcome.ReuseEligible || calls != 2 || assessments != 2 || nativePrepared != 2 || events != priorEvents || store.writes != 1 || len(keys) != 2 || keys[0] != keys[1] { t.Fatalf("settings/session reuse = %d %#v %v; calls=%d assessments=%d native-prepared=%d events=%d keys=%v", value, outcome, err, calls, assessments, nativePrepared, events, keys) } if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) { t.Fatalf("cache decisions = %v", decisions) } operation.Assess = acceptedProducerAssessment initial.Inputs["source"] = promptkit.Inline("changed source content") value, _, err = RunProducer(context.Background(), engine, operation, initial, interpretApplied, producerCorrection, nil) if err != nil || value != 20 || calls != 3 || len(keys) != 3 || keys[0] == keys[2] || store.writes != 2 { t.Fatalf("content change did not miss: value %d error %v calls %d keys %v", value, err, calls, keys) } } func TestProducerExecutesFrozenPreparedHandleAfterLookup(t *testing.T) { _, store := filesystemProducerCache(t) initial := nativeProducerInitial() engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if len(request.Prompt.Messages) != 2 || request.Prompt.Messages[1].Content != "Source: fixed source" { t.Fatalf("request was prepared again after cache lookup: %#v", request.Prompt.Messages) } return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := numberCacheOperation(store) var hash string operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error { if observation.Prepared != nil { hash = observation.MessageContentHash initial.Inputs["source"] = promptkit.Inline("mutation after preparation") observation.Prepared.Messages[1].Content = "mutated detached details" observation.Prepared.MessageContentHash = "not-the-key" } return nil } store.beforeWrite = func(key string, _ []byte) { if key != hash { t.Fatalf("mutable snapshot changed key: %s", key) } } if value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, nil, nil); err != nil || value != 1 { t.Fatalf("prepared execution = %d, %v", value, err) } } func TestProducerCacheSkipsIneligibleBypassedAndNonacceptedWrites(t *testing.T) { for _, test := range []struct { name string mode contracts.CacheMode eligible, reject, permissive, cancel, providerFailure, structuralFailure bool wantReads, wantWrites int }{ {name: "bypass", mode: contracts.CacheBypass, eligible: true}, {name: "ineligible", mode: contracts.CacheAuto}, {name: "refresh", mode: contracts.CacheRefresh, eligible: true, wantWrites: 1}, {name: "strict rejection", mode: contracts.CacheAuto, eligible: true, reject: true, wantReads: 1}, {name: "permissive fallback", mode: contracts.CacheAuto, eligible: true, reject: true, permissive: true, wantReads: 1}, {name: "cancellation", mode: contracts.CacheAuto, eligible: true, cancel: true, wantReads: 1}, {name: "provider failure", mode: contracts.CacheAuto, eligible: true, providerFailure: true, wantReads: 1}, {name: "structural failure", mode: contracts.CacheAuto, eligible: true, structuralFailure: true, wantReads: 1}, } { t.Run(test.name, func(t *testing.T) { _, store := filesystemProducerCache(t) providerFailure := errors.New("offline provider failure") engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if test.providerFailure { return nil, providerFailure } if test.structuralFailure { return &promptkit.GenerateResponse{Content: "not JSON"}, nil } return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) ctx, cancel := context.WithCancel(context.Background()) defer cancel() operation := numberCacheOperation(store) operation.Controls.Cache, operation.ReuseEligible = test.mode, test.eligible operation.Controls.MaxAdditionalTurns = 0 if test.permissive { operation.Controls.Mode = contracts.ModePermissive } operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { if test.cancel { cancel() } if test.reject { return rejectedProducerAssessment(ctx, value) } return acceptedProducerAssessment(ctx, value) } _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { return -1, nil }) switch { case test.cancel: if !errors.Is(err, context.Canceled) || outcome.Disposition != "" { t.Fatalf("canceled=%#v %v", outcome, err) } case test.providerFailure: if !errors.Is(err, providerFailure) { t.Fatalf("provider failure=%v", err) } case test.permissive: if err != nil || outcome.Disposition != contracts.ProducerFallback { t.Fatalf("fallback=%#v %v", outcome, err) } case test.reject || test.structuralFailure: if err == nil { t.Fatal("rejection accepted") } default: if err != nil || outcome.Disposition != contracts.ProducerAccepted { t.Fatalf("acceptance=%#v %v", outcome, err) } } if store.reads != test.wantReads || store.writes != test.wantWrites { t.Fatalf("cache reads/writes=%d/%d; want %d/%d", store.reads, store.writes, test.wantReads, test.wantWrites) } }) } } func TestProducerCacheMissesOnUnusableArtifactAndTreatsIOAsDiagnostic(t *testing.T) { for _, scenario := range []string{"unusable", "io failure"} { t.Run(scenario, func(t *testing.T) { _, store := filesystemProducerCache(t) calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := numberCacheOperation(store) var diagnostics []contracts.CacheObservation operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error { if observation.Prepared != nil && scenario == "unusable" { return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte(`{"wrong":"artifact"}`)) } if observation.Err != nil { diagnostics = append(diagnostics, observation) } return nil } if scenario == "io failure" { store.readErr = errors.New("unreadable cache") store.writeErr = errors.New("unwritable cache") } value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 1 || len(diagnostics) == 0 { t.Fatalf("cache diagnostic changed successful work: %d %#v %v, calls %d, diagnostics %v", value, outcome, err, calls, diagnostics) } }) } } func TestProducerCacheDoesNotHideEncodingOrCaptureFailureWithFallback(t *testing.T) { for _, boundary := range []string{"encode", "capture"} { t.Run(boundary, func(t *testing.T) { _, store := filesystemProducerCache(t) failure := errors.New(boundary + " failed") calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := numberCacheOperation(store) operation.Controls.Mode = contracts.ModePermissive if boundary == "encode" { operation.Cache.Encode = func(int) ([]byte, error) { return nil, failure } } else { operation.Cache.Observe = func(context.Context, contracts.CacheObservation) error { return failure } } fallbacks := 0 _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) if !errors.Is(err, failure) || fallbacks != 0 || store.writes != 0 || outcome.Disposition != "" { t.Fatalf("%s failure=%#v %v, fallback=%d", boundary, outcome, err, fallbacks) } if boundary == "capture" && calls != 0 { t.Fatal("execution started despite failed prepared capture") } }) } } func TestProducerCacheFailedRefreshPreservesPreviousArtifact(t *testing.T) { root, store := filesystemProducerCache(t) engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return nil, errors.New("refresh failed") })) prepared, err := engine.PrepareExecution(context.Background(), nativeProducerInitial()) if err != nil { t.Fatal(err) } key := prepared.Details().MessageContentHash prepared.Discard() if err := store.inner.Store(context.Background(), contracts.CacheAuto, key, []byte("7")); err != nil { t.Fatal(err) } operation := numberCacheOperation(store) operation.Controls.Cache = contracts.CacheRefresh if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err == nil { t.Fatal("refresh failure accepted") } content, err := os.ReadFile(filepath.Join(root, key+".json")) if err != nil || strings.TrimSpace(string(content)) != "7" || store.reads != 0 || store.writes != 0 { t.Fatalf("refresh destroyed prior acceptance: %q %v, reads/writes %d/%d", content, err, store.reads, store.writes) } }