345 lines
15 KiB
Go
345 lines
15 KiB
Go
|
|
package pipeline
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/artifactcache"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
type producerCacheStore struct {
|
||
|
|
inner contracts.ArtifactCache
|
||
|
|
reads, writes int
|
||
|
|
readErr, writeErr error
|
||
|
|
beforeWrite func(string, []byte)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (store *producerCacheStore) Lookup(ctx context.Context, mode contracts.CacheMode, hash string) ([]byte, error) {
|
||
|
|
store.reads++
|
||
|
|
if store.readErr != nil {
|
||
|
|
return nil, store.readErr
|
||
|
|
}
|
||
|
|
return store.inner.Lookup(ctx, mode, hash)
|
||
|
|
}
|
||
|
|
func (store *producerCacheStore) Store(ctx context.Context, mode contracts.CacheMode, hash string, content []byte) error {
|
||
|
|
store.writes++
|
||
|
|
if store.beforeWrite != nil {
|
||
|
|
store.beforeWrite(hash, content)
|
||
|
|
}
|
||
|
|
if store.writeErr != nil {
|
||
|
|
return store.writeErr
|
||
|
|
}
|
||
|
|
return store.inner.Store(ctx, mode, hash, content)
|
||
|
|
}
|
||
|
|
|
||
|
|
func numberCacheOperation(store contracts.ArtifactCache) contracts.ProducerOperation[int] {
|
||
|
|
return contracts.ProducerOperation[int]{
|
||
|
|
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheAuto, MaxAdditionalTurns: 1},
|
||
|
|
ReuseEligible: true, Assess: acceptedProducerAssessment,
|
||
|
|
Cache: &contracts.ProducerCache[int]{
|
||
|
|
Store: store, Encode: func(value int) ([]byte, error) { return json.Marshal(value) },
|
||
|
|
Decode: func(content []byte) (int, error) {
|
||
|
|
var value int
|
||
|
|
err := json.Unmarshal(content, &value)
|
||
|
|
return value, err
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func filesystemProducerCache(t *testing.T) (string, *producerCacheStore) {
|
||
|
|
t.Helper()
|
||
|
|
root := t.TempDir()
|
||
|
|
store, err := artifactcache.NewFilesystemStore(root)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
return root, &producerCacheStore{inner: store}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerCachesAppliedCorrectionUnderInitialKeyAndReusesWithoutExecution(t *testing.T) {
|
||
|
|
root, store := filesystemProducerCache(t)
|
||
|
|
calls, assessments, nativePrepared, events := 0, 0, 0, 0
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
value := 2
|
||
|
|
if calls == 0 {
|
||
|
|
value = 0
|
||
|
|
}
|
||
|
|
calls++
|
||
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
||
|
|
}))
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
operation.Execution = contracts.ExecutionOptions{
|
||
|
|
OnPrepared: func(context.Context, promptkit.PreparedRun) { nativePrepared++ },
|
||
|
|
Observer: func(promptkit.ExecutionEvent) { events++ },
|
||
|
|
}
|
||
|
|
var keys []string
|
||
|
|
var decisions []string
|
||
|
|
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
|
||
|
|
decisions = append(decisions, observation.Decision)
|
||
|
|
if observation.Prepared != nil {
|
||
|
|
if observation.Prepared.MessageContentHash == "" || len(observation.Prepared.Messages) == 0 {
|
||
|
|
t.Fatal("lost prepared details")
|
||
|
|
}
|
||
|
|
keys = append(keys, observation.MessageContentHash)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||
|
|
assessments++
|
||
|
|
if value == 20 {
|
||
|
|
return acceptedProducerAssessment(ctx, value)
|
||
|
|
}
|
||
|
|
return rejectedProducerAssessment(ctx, value)
|
||
|
|
}
|
||
|
|
// The response is a proposal; the cache must contain the applied typed value,
|
||
|
|
// not the private response or an earlier candidate.
|
||
|
|
interpretApplied := func(ctx context.Context, response *promptkit.RunResult) (*int, []string, error) {
|
||
|
|
value, guidance, err := interpretProducerNumber(ctx, response)
|
||
|
|
if value != nil {
|
||
|
|
*value *= 10
|
||
|
|
}
|
||
|
|
return value, guidance, err
|
||
|
|
}
|
||
|
|
store.beforeWrite = func(hash string, content []byte) {
|
||
|
|
if assessments != 2 || calls != 2 || string(content) != "20" || hash != keys[0] {
|
||
|
|
t.Fatalf("early or incorrect cache store: %s %q, calls=%d assessments=%d", hash, content, calls, assessments)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretApplied, producerCorrection, nil)
|
||
|
|
if err != nil || value != 20 || outcome.Disposition != contracts.ProducerAccepted || store.writes != 1 || nativePrepared != 2 || events == 0 {
|
||
|
|
t.Fatalf("accepted correction = %d %#v %v; writes %d, prepared %d, events %d", value, outcome, err, store.writes, nativePrepared, events)
|
||
|
|
}
|
||
|
|
content, err := os.ReadFile(filepath.Join(root, keys[0]+".json"))
|
||
|
|
if err != nil || string(content) != "20" {
|
||
|
|
t.Fatalf("stored artifact = %q, %v", content, err)
|
||
|
|
}
|
||
|
|
priorEvents := events
|
||
|
|
store.beforeWrite = nil
|
||
|
|
initial := nativeProducerInitial()
|
||
|
|
initial.SessionID, initial.ProfileID = "another-session", "chosen"
|
||
|
|
operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "another-model"}
|
||
|
|
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
|
||
|
|
t.Fatal("cache hit reassessed")
|
||
|
|
return contracts.CandidateAssessment{}, nil
|
||
|
|
}
|
||
|
|
value, outcome, err = RunProducer(context.Background(), engine, operation, initial,
|
||
|
|
func(context.Context, *promptkit.RunResult) (*int, []string, error) {
|
||
|
|
t.Fatal("cache hit interpreted a model result")
|
||
|
|
return nil, nil, nil
|
||
|
|
}, producerCorrection, nil)
|
||
|
|
if err != nil || value != 20 || !outcome.ReuseEligible || calls != 2 || assessments != 2 || nativePrepared != 2 || events != priorEvents || store.writes != 1 || len(keys) != 2 || keys[0] != keys[1] {
|
||
|
|
t.Fatalf("settings/session reuse = %d %#v %v; calls=%d assessments=%d native-prepared=%d events=%d keys=%v", value, outcome, err, calls, assessments, nativePrepared, events, keys)
|
||
|
|
}
|
||
|
|
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
|
||
|
|
t.Fatalf("cache decisions = %v", decisions)
|
||
|
|
}
|
||
|
|
operation.Assess = acceptedProducerAssessment
|
||
|
|
initial.Inputs["source"] = promptkit.Inline("changed source content")
|
||
|
|
value, _, err = RunProducer(context.Background(), engine, operation, initial, interpretApplied, producerCorrection, nil)
|
||
|
|
if err != nil || value != 20 || calls != 3 || len(keys) != 3 || keys[0] == keys[2] || store.writes != 2 {
|
||
|
|
t.Fatalf("content change did not miss: value %d error %v calls %d keys %v", value, err, calls, keys)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerExecutesFrozenPreparedHandleAfterLookup(t *testing.T) {
|
||
|
|
_, store := filesystemProducerCache(t)
|
||
|
|
initial := nativeProducerInitial()
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
if len(request.Prompt.Messages) != 2 || request.Prompt.Messages[1].Content != "Source: fixed source" {
|
||
|
|
t.Fatalf("request was prepared again after cache lookup: %#v", request.Prompt.Messages)
|
||
|
|
}
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||
|
|
}))
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
var hash string
|
||
|
|
operation.Cache.Observe = func(_ context.Context, observation contracts.CacheObservation) error {
|
||
|
|
if observation.Prepared != nil {
|
||
|
|
hash = observation.MessageContentHash
|
||
|
|
initial.Inputs["source"] = promptkit.Inline("mutation after preparation")
|
||
|
|
observation.Prepared.Messages[1].Content = "mutated detached details"
|
||
|
|
observation.Prepared.MessageContentHash = "not-the-key"
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
store.beforeWrite = func(key string, _ []byte) {
|
||
|
|
if key != hash {
|
||
|
|
t.Fatalf("mutable snapshot changed key: %s", key)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, nil, nil); err != nil || value != 1 {
|
||
|
|
t.Fatalf("prepared execution = %d, %v", value, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerCacheSkipsIneligibleBypassedAndNonacceptedWrites(t *testing.T) {
|
||
|
|
for _, test := range []struct {
|
||
|
|
name string
|
||
|
|
mode contracts.CacheMode
|
||
|
|
eligible, reject, permissive, cancel, providerFailure, structuralFailure bool
|
||
|
|
wantReads, wantWrites int
|
||
|
|
}{
|
||
|
|
{name: "bypass", mode: contracts.CacheBypass, eligible: true},
|
||
|
|
{name: "ineligible", mode: contracts.CacheAuto},
|
||
|
|
{name: "refresh", mode: contracts.CacheRefresh, eligible: true, wantWrites: 1},
|
||
|
|
{name: "strict rejection", mode: contracts.CacheAuto, eligible: true, reject: true, wantReads: 1},
|
||
|
|
{name: "permissive fallback", mode: contracts.CacheAuto, eligible: true, reject: true, permissive: true, wantReads: 1},
|
||
|
|
{name: "cancellation", mode: contracts.CacheAuto, eligible: true, cancel: true, wantReads: 1},
|
||
|
|
{name: "provider failure", mode: contracts.CacheAuto, eligible: true, providerFailure: true, wantReads: 1},
|
||
|
|
{name: "structural failure", mode: contracts.CacheAuto, eligible: true, structuralFailure: true, wantReads: 1},
|
||
|
|
} {
|
||
|
|
t.Run(test.name, func(t *testing.T) {
|
||
|
|
_, store := filesystemProducerCache(t)
|
||
|
|
providerFailure := errors.New("offline provider failure")
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
if test.providerFailure {
|
||
|
|
return nil, providerFailure
|
||
|
|
}
|
||
|
|
if test.structuralFailure {
|
||
|
|
return &promptkit.GenerateResponse{Content: "not JSON"}, nil
|
||
|
|
}
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||
|
|
}))
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
defer cancel()
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
operation.Controls.Cache, operation.ReuseEligible = test.mode, test.eligible
|
||
|
|
operation.Controls.MaxAdditionalTurns = 0
|
||
|
|
if test.permissive {
|
||
|
|
operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
}
|
||
|
|
operation.Assess = func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
||
|
|
if test.cancel {
|
||
|
|
cancel()
|
||
|
|
}
|
||
|
|
if test.reject {
|
||
|
|
return rejectedProducerAssessment(ctx, value)
|
||
|
|
}
|
||
|
|
return acceptedProducerAssessment(ctx, value)
|
||
|
|
}
|
||
|
|
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection,
|
||
|
|
func(context.Context, *int) (int, error) { return -1, nil })
|
||
|
|
switch {
|
||
|
|
case test.cancel:
|
||
|
|
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
|
||
|
|
t.Fatalf("canceled=%#v %v", outcome, err)
|
||
|
|
}
|
||
|
|
case test.providerFailure:
|
||
|
|
if !errors.Is(err, providerFailure) {
|
||
|
|
t.Fatalf("provider failure=%v", err)
|
||
|
|
}
|
||
|
|
case test.permissive:
|
||
|
|
if err != nil || outcome.Disposition != contracts.ProducerFallback {
|
||
|
|
t.Fatalf("fallback=%#v %v", outcome, err)
|
||
|
|
}
|
||
|
|
case test.reject || test.structuralFailure:
|
||
|
|
if err == nil {
|
||
|
|
t.Fatal("rejection accepted")
|
||
|
|
}
|
||
|
|
default:
|
||
|
|
if err != nil || outcome.Disposition != contracts.ProducerAccepted {
|
||
|
|
t.Fatalf("acceptance=%#v %v", outcome, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if store.reads != test.wantReads || store.writes != test.wantWrites {
|
||
|
|
t.Fatalf("cache reads/writes=%d/%d; want %d/%d", store.reads, store.writes, test.wantReads, test.wantWrites)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerCacheMissesOnUnusableArtifactAndTreatsIOAsDiagnostic(t *testing.T) {
|
||
|
|
for _, scenario := range []string{"unusable", "io failure"} {
|
||
|
|
t.Run(scenario, func(t *testing.T) {
|
||
|
|
_, store := filesystemProducerCache(t)
|
||
|
|
calls := 0
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
calls++
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||
|
|
}))
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
var diagnostics []contracts.CacheObservation
|
||
|
|
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
|
||
|
|
if observation.Prepared != nil && scenario == "unusable" {
|
||
|
|
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte(`{"wrong":"artifact"}`))
|
||
|
|
}
|
||
|
|
if observation.Err != nil {
|
||
|
|
diagnostics = append(diagnostics, observation)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
if scenario == "io failure" {
|
||
|
|
store.readErr = errors.New("unreadable cache")
|
||
|
|
store.writeErr = errors.New("unwritable cache")
|
||
|
|
}
|
||
|
|
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||
|
|
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 1 || len(diagnostics) == 0 {
|
||
|
|
t.Fatalf("cache diagnostic changed successful work: %d %#v %v, calls %d, diagnostics %v", value, outcome, err, calls, diagnostics)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerCacheDoesNotHideEncodingOrCaptureFailureWithFallback(t *testing.T) {
|
||
|
|
for _, boundary := range []string{"encode", "capture"} {
|
||
|
|
t.Run(boundary, func(t *testing.T) {
|
||
|
|
_, store := filesystemProducerCache(t)
|
||
|
|
failure := errors.New(boundary + " failed")
|
||
|
|
calls := 0
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
calls++
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||
|
|
}))
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
operation.Controls.Mode = contracts.ModePermissive
|
||
|
|
if boundary == "encode" {
|
||
|
|
operation.Cache.Encode = func(int) ([]byte, error) { return nil, failure }
|
||
|
|
} else {
|
||
|
|
operation.Cache.Observe = func(context.Context, contracts.CacheObservation) error { return failure }
|
||
|
|
}
|
||
|
|
fallbacks := 0
|
||
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
||
|
|
if !errors.Is(err, failure) || fallbacks != 0 || store.writes != 0 || outcome.Disposition != "" {
|
||
|
|
t.Fatalf("%s failure=%#v %v, fallback=%d", boundary, outcome, err, fallbacks)
|
||
|
|
}
|
||
|
|
if boundary == "capture" && calls != 0 {
|
||
|
|
t.Fatal("execution started despite failed prepared capture")
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestProducerCacheFailedRefreshPreservesPreviousArtifact(t *testing.T) {
|
||
|
|
root, store := filesystemProducerCache(t)
|
||
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
return nil, errors.New("refresh failed")
|
||
|
|
}))
|
||
|
|
prepared, err := engine.PrepareExecution(context.Background(), nativeProducerInitial())
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
key := prepared.Details().MessageContentHash
|
||
|
|
prepared.Discard()
|
||
|
|
if err := store.inner.Store(context.Background(), contracts.CacheAuto, key, []byte("7")); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
operation := numberCacheOperation(store)
|
||
|
|
operation.Controls.Cache = contracts.CacheRefresh
|
||
|
|
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err == nil {
|
||
|
|
t.Fatal("refresh failure accepted")
|
||
|
|
}
|
||
|
|
content, err := os.ReadFile(filepath.Join(root, key+".json"))
|
||
|
|
if err != nil || strings.TrimSpace(string(content)) != "7" || store.reads != 0 || store.writes != 0 {
|
||
|
|
t.Fatalf("refresh destroyed prior acceptance: %q %v, reads/writes %d/%d", content, err, store.reads, store.writes)
|
||
|
|
}
|
||
|
|
}
|