Cache final typed producer artifacts under the initial native request hash
This commit is contained in:
253
internal/framework/pipeline/runner_cache_test.go
Normal file
253
internal/framework/pipeline/runner_cache_test.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRunnerReusesAcceptedPredecessorAfterDownstreamFailure(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
probe := newRunnerProbe()
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "cached-producer", probe: probe, native: true})
|
||||
consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe})
|
||||
profile := typedResolutionProfile()
|
||||
profile.Artifacts = nil
|
||||
profile.Steps = []PipelineStepProfile{
|
||||
{ID: "produce", Artifacts: map[string]ArtifactLaneProfile{"producer": producer}},
|
||||
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer}},
|
||||
}
|
||||
prepared := prepareRunnerProfile(t, catalog, profile, engine)
|
||||
failure := errors.New("downstream application failure")
|
||||
first := true
|
||||
consumerOperation := prepared.Steps[1].lanes[0].typed.extract
|
||||
prepared.Steps[1].lanes[0].typed.extract = func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
if first {
|
||||
return erasedTypedResult{}, failure
|
||||
}
|
||||
return consumerOperation(ctx, implementation, request)
|
||||
}
|
||||
var observations []contracts.CacheObservation
|
||||
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Cache: store,
|
||||
ObserveCache: func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
|
||||
if origin.ModuleKey != "cached-producer" || origin.Stage != contracts.DiagnosticOriginStageExtract || observation.ChunkID == "" || observation.ChunkIndex == nil {
|
||||
t.Fatalf("lost cache scope: %#v %#v", origin, observation)
|
||||
}
|
||||
observations = append(observations, observation)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
if _, err := New().Run(context.Background(), input); !errors.Is(err, failure) {
|
||||
t.Fatalf("downstream failure=%v", err)
|
||||
}
|
||||
if calls != 1 || store.reads != 1 || store.writes != 1 {
|
||||
t.Fatalf("predecessor not stored before downstream failure: calls %d reads/writes %d/%d", calls, store.reads, store.writes)
|
||||
}
|
||||
first = false
|
||||
output, err := New().Run(context.Background(), input)
|
||||
if err != nil || len(output.NormalizeOutputs) != 2 || calls != 1 || store.reads != 2 || store.writes != 1 || output.FallbackCount != 0 {
|
||||
t.Fatalf("rerun=%v outputs=%d calls=%d cache=%d/%d", err, len(output.NormalizeOutputs), calls, store.reads, store.writes)
|
||||
}
|
||||
var decisions []string
|
||||
for _, observation := range observations {
|
||||
decisions = append(decisions, observation.Decision)
|
||||
}
|
||||
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
|
||||
t.Fatalf("cache decisions=%v", decisions)
|
||||
}
|
||||
// The deterministic merge/normalize work runs normally on both invocations;
|
||||
// the reused producer's semantic assessor is not repeated.
|
||||
if probe.calls["cached-producer/merge"] != 2 || probe.calls["cached-producer/normalize"] != 2 {
|
||||
t.Fatalf("deterministic work was incorrectly cached: %v", probe.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCacheWarningsDoNotFailSuccessfulPublication(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
store.readErr = errors.New("cache read unavailable")
|
||||
store.writeErr = errors.New(strings.Repeat("cache write unavailable ", 400))
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
profile := typedResolutionProfile()
|
||||
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), Cache: store, Workers: 1})
|
||||
if err != nil || len(output.NormalizeOutputs) != 1 || output.FallbackCount != 0 {
|
||||
t.Fatalf("cache warnings failed successful work: %v, outputs=%d", err, len(output.NormalizeOutputs))
|
||||
}
|
||||
projection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
||||
if err != nil || projection.WarningOccurrenceCount != 2 {
|
||||
t.Fatalf("cache diagnostics=%#v %v", projection, err)
|
||||
}
|
||||
for _, warning := range projection.Warnings {
|
||||
if warning.Origin.ModuleKey != "native" || warning.Origin.LaneID != "notes" || warning.Disposition != contracts.DiagnosticDispositionWarning {
|
||||
t.Fatalf("unscoped cache warning=%#v", warning)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedCacheDecodeUsesCodecAndCurrentSourceWithoutSemanticAssessment(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
doc := typedTestDocument()
|
||||
codec := notesCodec()
|
||||
registry := NewArtifactCodecRegistry()
|
||||
if err := RegisterArtifactCodec(registry, codec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, _, err := registry.entry("test/notes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
evidence := NewArtifactEvidenceRegistry()
|
||||
if err := RegisterArtifactEvidence(evidence, "test/notes", func(value codecNotes) []source.SourceRef {
|
||||
if value.Details == nil {
|
||||
return nil
|
||||
}
|
||||
return []source.SourceRef{{SourceID: value.Details.Name, StartUnitID: 1, EndUnitID: 1}}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
projector, _, err := evidence.entry("test/notes")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
typed := &preparedTypedLane{codec: entry, cacheEvidence: projector.project}
|
||||
var warnings []contracts.ProducerDiagnostic
|
||||
cache := artifactProducerCache(RunInput{Cache: store}, typed, contracts.ExecutionClassLLMBacked,
|
||||
contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, ModuleKey: "native"}, doc, nil, &warnings)
|
||||
specialized := specializeProducerOperation[codecNotes](contracts.ProducerOperation[any]{Cache: cache}).Cache
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
content []byte
|
||||
valid bool
|
||||
}{
|
||||
{"current source", []byte(`{"items":["accepted"],"details":{"name":"source"}}`), true},
|
||||
{"stale source", []byte(`{"items":["accepted"],"details":{"name":"old-source"}}`), false},
|
||||
{"unusable codec", []byte(`{"items":"not a collection"}`), false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
value, err := specialized.Decode(test.content)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("decoded=%#v %v", value, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
// A codec mismatch stays a typed error at the registry boundary.
|
||||
wrong := specializeProducerOperation[codecScore](contracts.ProducerOperation[any]{Cache: cache}).Cache
|
||||
if _, err := wrong.Decode([]byte(`{"items":[]}`)); err == nil {
|
||||
t.Fatal("cache decode crossed an incompatible Go type")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProducerCacheCancellationDuringHitCannotDeliverAcceptance(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
t.Fatal("cache hit made a provider call")
|
||||
return nil, nil
|
||||
}))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
operation := numberCacheOperation(store)
|
||||
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
|
||||
if observation.Prepared != nil {
|
||||
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte("1"))
|
||||
}
|
||||
if observation.Decision == "hit" {
|
||||
cancel()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
||||
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
|
||||
t.Fatalf("canceled hit delivered acceptance: %#v %v", outcome, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPreparesCacheEvidenceWithoutEnablingEvidenceOutput(t *testing.T) {
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
catalog.ArtifactEvidence = NewArtifactEvidenceRegistry()
|
||||
if err := RegisterArtifactEvidence(catalog.ArtifactEvidence, "test/notes", func(value codecNotes) []source.SourceRef {
|
||||
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
|
||||
for _, lane := range prepared.Steps[0].lanes {
|
||||
if lane.resolved.ArtifactKind == "test/notes" {
|
||||
if lane.typed.cacheEvidence == nil {
|
||||
t.Fatal("source usability depended on evidence-context publication")
|
||||
}
|
||||
content, _ := json.Marshal(codecNotes{Items: []string{}})
|
||||
_, store := filesystemProducerCache(t)
|
||||
var warnings []contracts.ProducerDiagnostic
|
||||
cache := artifactProducerCache(RunInput{Cache: store}, lane.typed, contracts.ExecutionClassLLMBacked, contracts.DiagnosticOrigin{}, typedTestDocument(), nil, &warnings)
|
||||
if _, err := cache.Decode(content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerCachesAppliedMergeAndNormalizationArtifacts(t *testing.T) {
|
||||
for _, role := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
t.Run(string(role), func(t *testing.T) {
|
||||
_, store := filesystemProducerCache(t)
|
||||
calls := 0
|
||||
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
||||
}))
|
||||
prepared := preparedConcurrentPipeline(t, 1)
|
||||
lane := &prepared.Steps[0].lanes[0]
|
||||
produce := func(ctx context.Context, operation contracts.ProducerOperation[any]) (erasedTypedResult, error) {
|
||||
value, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
|
||||
func(context.Context, *promptkit.RunResult) (*any, []string, error) {
|
||||
var applied any = codecNotes{Items: []string{"applied"}}
|
||||
return &applied, nil, nil
|
||||
}, nil, nil)
|
||||
return erasedTypedResult{Value: value, Outcome: outcome}, err
|
||||
}
|
||||
if role == StageMerge {
|
||||
lane.resolved.MergeExecutionClass = contracts.ExecutionClassLLMBacked
|
||||
lane.typed.merge = func(ctx context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
||||
return produce(ctx, request.Operation)
|
||||
}
|
||||
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return erasedTypedResult{Value: request.MergeOutput.Value}, ctx.Err()
|
||||
}
|
||||
} else {
|
||||
lane.resolved.NormalizeExecutionClass = contracts.ExecutionClassLLMBacked
|
||||
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
||||
return produce(ctx, request.Operation)
|
||||
}
|
||||
}
|
||||
for range 2 {
|
||||
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: store, RawInput: []byte("input"), Workers: 1})
|
||||
if err != nil || len(output.NormalizeOutputs) != 2 {
|
||||
t.Fatalf("cached %s execution: %v, outputs %d", role, err, len(output.NormalizeOutputs))
|
||||
}
|
||||
value, err := notesCodec().Decode(output.NormalizeOutputs[0].Artifact.Content)
|
||||
if err != nil || !reflect.DeepEqual(value.Items, []string{"applied"}) {
|
||||
t.Fatalf("applied artifact changed on reuse: %#v, %v", value, err)
|
||||
}
|
||||
}
|
||||
if calls != 1 || store.reads != 2 || store.writes != 1 {
|
||||
t.Fatalf("%s cache calls: provider %d, reads/writes %d/%d", role, calls, store.reads, store.writes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user