Files
notarius/internal/framework/pipeline/producer_cache_test.go

345 lines
15 KiB
Go

package pipeline
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type producerCacheStore struct {
inner contracts.ArtifactCache
reads, writes int
readErr, writeErr error
beforeWrite func(string, []byte)
}
func (store *producerCacheStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) {
store.reads++
if store.readErr != nil {
return nil, store.readErr
}
return store.inner.Lookup(ctx, mode, hash)
}
func (store *producerCacheStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error {
store.writes++
if store.beforeWrite != nil {
store.beforeWrite(hash, content)
}
if store.writeErr != nil {
return store.writeErr
}
return store.inner.Store(ctx, mode, hash, content)
}
func numberCacheOperation(store contracts.ArtifactCache) contracts.ProducerOperation[int] {
return contracts.ProducerOperation[int]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheAuto, MaxAdditionalTurns: 1},
ReuseEligible: true, Assess: acceptedProducerAssessment,
Cache: &contracts.ProducerCache[int]{
Store: store, Encode: func(value int) ([]byte, error) { return json.Marshal(value) },
Decode: func(content []byte) (int, error) {
var value int
err := json.Unmarshal(content, &value)
return value, err
},
},
}
}
func filesystemProducerCache(t *testing.T) (string, *producerCacheStore) {
t.Helper()
root := t.TempDir()
store, err := artifactcache.NewFilesystemStore(root)
if err != nil {
t.Fatal(err)
}
return root, &producerCacheStore{inner: store}
}
func TestProducerCachesAppliedCorrectionUnderInitialKeyAndReusesWithoutExecution(t *testing.T) {
root, store := filesystemProducerCache(t)
calls, assessments, nativePrepared, events := 0, 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
value := 2
if calls == 0 {
value = 0
}
calls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
operation := numberCacheOperation(store)
operation.Execution = contracts.ExecutionOptions{
OnPrepared: func(context.Context, promptkit.PreparedRun) { nativePrepared++ },
Observer: func(promptkit.ExecutionEvent) { events++ },
}
var keys []string
var decisions []string
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
decisions = append(decisions, observation.Decision)
if observation.Prepared != nil {
if observation.Prepared.MessageContentHash == "" || len(observation.Prepared.Messages) == 0 {
t.Fatal("lost prepared details")
}
keys = append(keys, observation.MessageContentHash)
}
return nil
}
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessments++
if value == 20 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
}
// The response is a proposal; the cache must contain the applied typed value,
// not the private response or an earlier candidate.
interpretApplied := func(ctx context.Context, response *promptkit.RunResult) (*int, []string, error) {
value, guidance, err := interpretProducerNumber(ctx, response)
if value != nil {
*value *= 10
}
return value, guidance, err
}
store.beforeWrite = func(hash string, content []byte) {
if assessments != 2 || calls != 2 || string(content) != "20" || hash != keys[0] {
t.Fatalf("early or incorrect cache store: %s %q, calls=%d assessments=%d", hash, content, calls, assessments)
}
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretApplied, producerCorrection, nil)
if err != nil || value != 20 || outcome.Disposition != contracts.ProducerAccepted || store.writes != 1 || nativePrepared != 2 || events == 0 {
t.Fatalf("accepted correction = %d %#v %v; writes %d, prepared %d, events %d", value, outcome, err, store.writes, nativePrepared, events)
}
content, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
if err != nil || string(content) != "20" {
t.Fatalf("stored artifact = %q, %v", content, err)
}
priorEvents := events
store.beforeWrite = nil
initial := nativeProducerInitial()
initial.SessionID, initial.ProfileID = "another-session", "chosen"
operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
t.Fatal("cache hit reassessed")
return contracts.CandidateAssessment{}, nil
}
value, outcome, err = RunProducer(context.Background(), engine, operation, initial,
func(context.Context, *promptkit.RunResult) (*int, []string, error) {
t.Fatal("cache hit interpreted a model result")
return nil, nil, nil
}, producerCorrection, nil)
if err != nil || value != 20 || !outcome.ReuseEligible || calls != 2 || assessments != 2 || nativePrepared != 2 || events != priorEvents || store.writes != 1 || len(keys) != 2 || keys[0] != keys[1] {
t.Fatalf("settings/session reuse = %d %#v %v; calls=%d assessments=%d native-prepared=%d events=%d keys=%v", value, outcome, err, calls, assessments, nativePrepared, events, keys)
}
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
t.Fatalf("cache decisions = %v", decisions)
}
operation.Assess = acceptedProducerAssessment
initial.Inputs["source"] = promptkit.Inline("changed source content")
value, _, err = RunProducer(context.Background(), engine, operation, initial, interpretApplied, producerCorrection, nil)
if err != nil || value != 20 || calls != 3 || len(keys) != 3 || keys[0] == keys[2] || store.writes != 2 {
t.Fatalf("content change did not miss: value %d error %v calls %d keys %v", value, err, calls, keys)
}
}
func TestProducerExecutesFrozenPreparedHandleAfterLookup(t *testing.T) {
_, store := filesystemProducerCache(t)
initial := nativeProducerInitial()
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if len(request.Prompt.Messages) != 2 || request.Prompt.Messages[1].Content != "Source: fixed source" {
t.Fatalf("request was prepared again after cache lookup: %#v", request.Prompt.Messages)
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
var hash string
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
if observation.Prepared != nil {
hash = observation.MessageContentHash
initial.Inputs["source"] = promptkit.Inline("mutation after preparation")
observation.Prepared.Messages[1].Content = "mutated detached details"
observation.Prepared.MessageContentHash = "not-the-key"
}
return nil
}
store.beforeWrite = func(key string, _ []byte) {
if key != hash {
t.Fatalf("mutable snapshot changed key: %s", key)
}
}
if value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, nil, nil); err != nil || value != 1 {
t.Fatalf("prepared execution = %d, %v", value, err)
}
}
func TestProducerCacheSkipsIneligibleBypassedAndNonacceptedWrites(t *testing.T) {
for _, test := range []struct {
name string
mode contracts.CacheMode
eligible, reject, permissive, cancel, providerFailure, structuralFailure bool
wantReads, wantWrites int
}{
{name: "bypass", mode: contracts.CacheBypass, eligible: true},
{name: "ineligible", mode: contracts.CacheAuto},
{name: "refresh", mode: contracts.CacheRefresh, eligible: true, wantWrites: 1},
{name: "strict rejection", mode: contracts.CacheAuto, eligible: true, reject: true, wantReads: 1},
{name: "permissive fallback", mode: contracts.CacheAuto, eligible: true, reject: true, permissive: true, wantReads: 1},
{name: "cancellation", mode: contracts.CacheAuto, eligible: true, cancel: true, wantReads: 1},
{name: "provider failure", mode: contracts.CacheAuto, eligible: true, providerFailure: true, wantReads: 1},
{name: "structural failure", mode: contracts.CacheAuto, eligible: true, structuralFailure: true, wantReads: 1},
} {
t.Run(test.name, func(t *testing.T) {
_, store := filesystemProducerCache(t)
providerFailure := errors.New("offline provider failure")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if test.providerFailure {
return nil, providerFailure
}
if test.structuralFailure {
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
}
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
operation := numberCacheOperation(store)
operation.Controls.Cache, operation.ReuseEligible = test.mode, test.eligible
operation.Controls.MaxAdditionalTurns = 0
if test.permissive {
operation.Controls.Mode = contracts.ModePermissive
}
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
if test.cancel {
cancel()
}
if test.reject {
return rejectedProducerAssessment(ctx, value)
}
return acceptedProducerAssessment(ctx, value)
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection,
func(context.Context, *int) (int, error) { return -1, nil })
switch {
case test.cancel:
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
t.Fatalf("canceled=%#v %v", outcome, err)
}
case test.providerFailure:
if !errors.Is(err, providerFailure) {
t.Fatalf("provider failure=%v", err)
}
case test.permissive:
if err != nil || outcome.Disposition != contracts.ProducerFallback {
t.Fatalf("fallback=%#v %v", outcome, err)
}
case test.reject || test.structuralFailure:
if err == nil {
t.Fatal("rejection accepted")
}
default:
if err != nil || outcome.Disposition != contracts.ProducerAccepted {
t.Fatalf("acceptance=%#v %v", outcome, err)
}
}
if store.reads != test.wantReads || store.writes != test.wantWrites {
t.Fatalf("cache reads/writes=%d/%d; want %d/%d", store.reads, store.writes, test.wantReads, test.wantWrites)
}
})
}
}
func TestProducerCacheMissesOnUnusableArtifactAndTreatsIOAsDiagnostic(t *testing.T) {
for _, scenario := range []string{"unusable", "io failure"} {
t.Run(scenario, func(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
var diagnostics []contracts.CacheObservation
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
if observation.Prepared != nil && scenario == "unusable" {
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte(`{"wrong":"artifact"}`))
}
if observation.Err != nil {
diagnostics = append(diagnostics, observation)
}
return nil
}
if scenario == "io failure" {
store.readErr = errors.New("unreadable cache")
store.writeErr = errors.New("unwritable cache")
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 1 || len(diagnostics) == 0 {
t.Fatalf("cache diagnostic changed successful work: %d %#v %v, calls %d, diagnostics %v", value, outcome, err, calls, diagnostics)
}
})
}
}
func TestProducerCacheDoesNotHideEncodingOrCaptureFailureWithFallback(t *testing.T) {
for _, boundary := range []string{"encode", "capture"} {
t.Run(boundary, func(t *testing.T) {
_, store := filesystemProducerCache(t)
failure := errors.New(boundary + " failed")
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
operation.Controls.Mode = contracts.ModePermissive
if boundary == "encode" {
operation.Cache.Encode = func(int) ([]byte, error) { return nil, failure }
} else {
operation.Cache.Observe = func(context.Context, contracts.CacheObservation) error { return failure }
}
fallbacks := 0
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, failure) || fallbacks != 0 || store.writes != 0 || outcome.Disposition != "" {
t.Fatalf("%s failure=%#v %v, fallback=%d", boundary, outcome, err, fallbacks)
}
if boundary == "capture" && calls != 0 {
t.Fatal("execution started despite failed prepared capture")
}
})
}
}
func TestProducerCacheFailedRefreshPreservesPreviousArtifact(t *testing.T) {
root, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, errors.New("refresh failed")
}))
prepared, err := engine.PrepareExecution(context.Background(), nativeProducerInitial())
if err != nil {
t.Fatal(err)
}
key := prepared.Details().MessageContentHash
prepared.Discard()
if err := store.inner.Store(context.Background(), contracts.CacheAuto, key, []byte("7")); err != nil {
t.Fatal(err)
}
operation := numberCacheOperation(store)
operation.Controls.Cache = contracts.CacheRefresh
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err == nil {
t.Fatal("refresh failure accepted")
}
content, err := os.ReadFile(filepath.Join(root, key+".json"))
if err != nil || strings.TrimSpace(string(content)) != "7" || store.reads != 0 || store.writes != 0 {
t.Fatalf("refresh destroyed prior acceptance: %q %v, reads/writes %d/%d", content, err, store.reads, store.writes)
}
}