2026-11-22 02:10:05 +00:00
|
|
|
package pipeline
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"errors"
|
|
|
|
|
"reflect"
|
|
|
|
|
"strings"
|
2026-11-22 09:42:06 +00:00
|
|
|
"sync/atomic"
|
2026-11-22 02:10:05 +00:00
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
|
|
|
"go.maximumdirect.net/promptkit"
|
|
|
|
|
)
|
|
|
|
|
|
2026-11-22 09:42:06 +00:00
|
|
|
func TestRunnerChecksCurrentSourceOnConcurrentCacheHits(t *testing.T) {
|
|
|
|
|
_, cache := filesystemProducerCache(t)
|
|
|
|
|
var calls atomic.Int32
|
|
|
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
calls.Add(1)
|
|
|
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
|
|
|
}))
|
|
|
|
|
const chunkCount = 4
|
|
|
|
|
prepared := preparedConcurrentPipeline(t, chunkCount)
|
|
|
|
|
for i := range prepared.Steps[0].lanes {
|
|
|
|
|
lane := &prepared.Steps[0].lanes[i]
|
|
|
|
|
if lane.resolved.ArtifactKind != "test/notes" {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
lane.resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked
|
|
|
|
|
lane.typed.cacheEvidence = func(any) ([]source.SourceRef, error) {
|
|
|
|
|
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil
|
|
|
|
|
}
|
|
|
|
|
lane.typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
|
|
|
|
initial := nativeProducerInitial()
|
|
|
|
|
initial.Inputs["source"] = promptkit.Inline(string(request.Chunk.Content))
|
|
|
|
|
value, outcome, err := RunProducer(ctx, engine, request.Operation, initial,
|
|
|
|
|
func(context.Context, *promptkit.RunResult) (*any, []string, error) {
|
|
|
|
|
var value any = codecNotes{Items: []string{"accepted"}}
|
|
|
|
|
return &value, nil, nil
|
|
|
|
|
}, nil, nil)
|
|
|
|
|
return erasedTypedResult{Value: value, Outcome: outcome}, err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for run := 0; run < 2; run++ {
|
|
|
|
|
// The filesystem store is concurrency-safe; its counting test wrapper is not.
|
|
|
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: cache.inner, RawInput: []byte("input"), Workers: chunkCount})
|
|
|
|
|
if err != nil || len(output.normalizeOutputs) != 2 {
|
|
|
|
|
t.Fatalf("run %d: outputs=%d error=%v", run, len(output.normalizeOutputs), err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if calls.Load() != chunkCount {
|
|
|
|
|
t.Fatalf("cache hits executed model work: calls=%d", calls.Load())
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-22 02:10:05 +00:00
|
|
|
func TestRunnerReusesAcceptedPredecessorAfterDownstreamFailure(t *testing.T) {
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
probe := newRunnerProbe()
|
|
|
|
|
calls := 0
|
|
|
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
calls++
|
|
|
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
|
|
|
}))
|
|
|
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
|
|
|
producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "cached-producer", probe: probe, native: true})
|
|
|
|
|
consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe})
|
|
|
|
|
profile := typedResolutionProfile()
|
|
|
|
|
profile.Artifacts = nil
|
|
|
|
|
profile.Steps = []PipelineStepProfile{
|
|
|
|
|
{ID: "produce", Artifacts: map[string]ArtifactLaneProfile{"producer": producer}},
|
|
|
|
|
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer}},
|
|
|
|
|
}
|
|
|
|
|
prepared := prepareRunnerProfile(t, catalog, profile, engine)
|
|
|
|
|
failure := errors.New("downstream application failure")
|
|
|
|
|
first := true
|
|
|
|
|
consumerOperation := prepared.Steps[1].lanes[0].typed.extract
|
|
|
|
|
prepared.Steps[1].lanes[0].typed.extract = func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
|
|
|
|
if first {
|
|
|
|
|
return erasedTypedResult{}, failure
|
|
|
|
|
}
|
|
|
|
|
return consumerOperation(ctx, implementation, request)
|
|
|
|
|
}
|
|
|
|
|
var observations []contracts.CacheObservation
|
|
|
|
|
input := RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Cache: store,
|
|
|
|
|
ObserveCache: func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error {
|
|
|
|
|
if origin.ModuleKey != "cached-producer" || origin.Stage != contracts.DiagnosticOriginStageExtract || observation.ChunkID == "" || observation.ChunkIndex == nil {
|
|
|
|
|
t.Fatalf("lost cache scope: %#v %#v", origin, observation)
|
|
|
|
|
}
|
|
|
|
|
observations = append(observations, observation)
|
|
|
|
|
return nil
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
if _, err := New().Run(context.Background(), input); !errors.Is(err, failure) {
|
|
|
|
|
t.Fatalf("downstream failure=%v", err)
|
|
|
|
|
}
|
|
|
|
|
if calls != 1 || store.reads != 1 || store.writes != 1 {
|
|
|
|
|
t.Fatalf("predecessor not stored before downstream failure: calls %d reads/writes %d/%d", calls, store.reads, store.writes)
|
|
|
|
|
}
|
|
|
|
|
first = false
|
|
|
|
|
output, err := New().Run(context.Background(), input)
|
2026-11-22 02:34:40 +00:00
|
|
|
if err != nil || len(output.normalizeOutputs) != 2 || calls != 1 || store.reads != 2 || store.writes != 1 || output.FallbackCount != 0 {
|
|
|
|
|
t.Fatalf("rerun=%v outputs=%d calls=%d cache=%d/%d", err, len(output.normalizeOutputs), calls, store.reads, store.writes)
|
2026-11-22 02:10:05 +00:00
|
|
|
}
|
|
|
|
|
var decisions []string
|
|
|
|
|
for _, observation := range observations {
|
|
|
|
|
decisions = append(decisions, observation.Decision)
|
|
|
|
|
}
|
|
|
|
|
if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) {
|
|
|
|
|
t.Fatalf("cache decisions=%v", decisions)
|
|
|
|
|
}
|
|
|
|
|
// The deterministic merge/normalize work runs normally on both invocations;
|
|
|
|
|
// the reused producer's semantic assessor is not repeated.
|
|
|
|
|
if probe.calls["cached-producer/merge"] != 2 || probe.calls["cached-producer/normalize"] != 2 {
|
|
|
|
|
t.Fatalf("deterministic work was incorrectly cached: %v", probe.calls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRunnerCacheWarningsDoNotFailSuccessfulPublication(t *testing.T) {
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
store.readErr = errors.New("cache read unavailable")
|
|
|
|
|
store.writeErr = errors.New(strings.Repeat("cache write unavailable ", 400))
|
|
|
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
|
|
|
}))
|
|
|
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
|
|
|
profile := typedResolutionProfile()
|
|
|
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
|
|
|
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), Cache: store, Workers: 1})
|
2026-11-22 02:34:40 +00:00
|
|
|
if err != nil || len(output.normalizeOutputs) != 1 || output.FallbackCount != 0 {
|
|
|
|
|
t.Fatalf("cache warnings failed successful work: %v, outputs=%d", err, len(output.normalizeOutputs))
|
2026-11-22 02:10:05 +00:00
|
|
|
}
|
|
|
|
|
projection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics)
|
|
|
|
|
if err != nil || projection.WarningOccurrenceCount != 2 {
|
|
|
|
|
t.Fatalf("cache diagnostics=%#v %v", projection, err)
|
|
|
|
|
}
|
|
|
|
|
for _, warning := range projection.Warnings {
|
|
|
|
|
if warning.Origin.ModuleKey != "native" || warning.Origin.LaneID != "notes" || warning.Disposition != contracts.DiagnosticDispositionWarning {
|
|
|
|
|
t.Fatalf("unscoped cache warning=%#v", warning)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestTypedCacheDecodeUsesCodecAndCurrentSourceWithoutSemanticAssessment(t *testing.T) {
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
doc := typedTestDocument()
|
|
|
|
|
codec := notesCodec()
|
|
|
|
|
registry := NewArtifactCodecRegistry()
|
|
|
|
|
if err := RegisterArtifactCodec(registry, codec); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
entry, _, err := registry.entry("test/notes")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
evidence := NewArtifactEvidenceRegistry()
|
|
|
|
|
if err := RegisterArtifactEvidence(evidence, "test/notes", func(value codecNotes) []source.SourceRef {
|
|
|
|
|
if value.Details == nil {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
return []source.SourceRef{{SourceID: value.Details.Name, StartUnitID: 1, EndUnitID: 1}}
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
projector, _, err := evidence.entry("test/notes")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
typed := &preparedTypedLane{codec: entry, cacheEvidence: projector.project}
|
|
|
|
|
var warnings []contracts.ProducerDiagnostic
|
|
|
|
|
cache := artifactProducerCache(RunInput{Cache: store}, typed, contracts.ExecutionClassLLMBacked,
|
|
|
|
|
contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, ModuleKey: "native"}, doc, nil, &warnings)
|
|
|
|
|
specialized := specializeProducerOperation[codecNotes](contracts.ProducerOperation[any]{Cache: cache}).Cache
|
|
|
|
|
for _, test := range []struct {
|
|
|
|
|
name string
|
|
|
|
|
content []byte
|
|
|
|
|
valid bool
|
|
|
|
|
}{
|
|
|
|
|
{"current source", []byte(`{"items":["accepted"],"details":{"name":"source"}}`), true},
|
|
|
|
|
{"stale source", []byte(`{"items":["accepted"],"details":{"name":"old-source"}}`), false},
|
|
|
|
|
{"unusable codec", []byte(`{"items":"not a collection"}`), false},
|
|
|
|
|
} {
|
|
|
|
|
t.Run(test.name, func(t *testing.T) {
|
|
|
|
|
value, err := specialized.Decode(test.content)
|
|
|
|
|
if (err == nil) != test.valid {
|
|
|
|
|
t.Fatalf("decoded=%#v %v", value, err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
// A codec mismatch stays a typed error at the registry boundary.
|
|
|
|
|
wrong := specializeProducerOperation[codecScore](contracts.ProducerOperation[any]{Cache: cache}).Cache
|
|
|
|
|
if _, err := wrong.Decode([]byte(`{"items":[]}`)); err == nil {
|
|
|
|
|
t.Fatal("cache decode crossed an incompatible Go type")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestProducerCacheCancellationDuringHitCannotDeliverAcceptance(t *testing.T) {
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
t.Fatal("cache hit made a provider call")
|
|
|
|
|
return nil, nil
|
|
|
|
|
}))
|
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
|
defer cancel()
|
|
|
|
|
operation := numberCacheOperation(store)
|
|
|
|
|
operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
|
|
|
|
|
if observation.Prepared != nil {
|
|
|
|
|
return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte("1"))
|
|
|
|
|
}
|
|
|
|
|
if observation.Decision == "hit" {
|
|
|
|
|
cancel()
|
|
|
|
|
}
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
|
|
|
|
if !errors.Is(err, context.Canceled) || outcome.Disposition != "" {
|
|
|
|
|
t.Fatalf("canceled hit delivered acceptance: %#v %v", outcome, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRunnerPreparesCacheEvidenceWithoutEnablingEvidenceOutput(t *testing.T) {
|
|
|
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
|
|
|
catalog.ArtifactEvidence = NewArtifactEvidenceRegistry()
|
|
|
|
|
if err := RegisterArtifactEvidence(catalog.ArtifactEvidence, "test/notes", func(value codecNotes) []source.SourceRef {
|
|
|
|
|
return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}
|
|
|
|
|
}); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil)
|
|
|
|
|
for _, lane := range prepared.Steps[0].lanes {
|
|
|
|
|
if lane.resolved.ArtifactKind == "test/notes" {
|
|
|
|
|
if lane.typed.cacheEvidence == nil {
|
|
|
|
|
t.Fatal("source usability depended on evidence-context publication")
|
|
|
|
|
}
|
|
|
|
|
content, _ := json.Marshal(codecNotes{Items: []string{}})
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
var warnings []contracts.ProducerDiagnostic
|
|
|
|
|
cache := artifactProducerCache(RunInput{Cache: store}, lane.typed, contracts.ExecutionClassLLMBacked, contracts.DiagnosticOrigin{}, typedTestDocument(), nil, &warnings)
|
|
|
|
|
if _, err := cache.Decode(content); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRunnerCachesAppliedMergeAndNormalizationArtifacts(t *testing.T) {
|
|
|
|
|
for _, role := range []ModuleStage{StageMerge, StageNormalize} {
|
|
|
|
|
t.Run(string(role), func(t *testing.T) {
|
|
|
|
|
_, store := filesystemProducerCache(t)
|
|
|
|
|
calls := 0
|
|
|
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
|
|
|
calls++
|
|
|
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
|
|
|
}))
|
|
|
|
|
prepared := preparedConcurrentPipeline(t, 1)
|
|
|
|
|
lane := &prepared.Steps[0].lanes[0]
|
|
|
|
|
produce := func(ctx context.Context, operation contracts.ProducerOperation[any]) (erasedTypedResult, error) {
|
|
|
|
|
value, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(),
|
|
|
|
|
func(context.Context, *promptkit.RunResult) (*any, []string, error) {
|
|
|
|
|
var applied any = codecNotes{Items: []string{"applied"}}
|
|
|
|
|
return &applied, nil, nil
|
|
|
|
|
}, nil, nil)
|
|
|
|
|
return erasedTypedResult{Value: value, Outcome: outcome}, err
|
|
|
|
|
}
|
|
|
|
|
if role == StageMerge {
|
|
|
|
|
lane.resolved.MergeExecutionClass = contracts.ExecutionClassLLMBacked
|
|
|
|
|
lane.typed.merge = func(ctx context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) {
|
|
|
|
|
return produce(ctx, request.Operation)
|
|
|
|
|
}
|
|
|
|
|
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
|
|
|
|
return erasedTypedResult{Value: request.MergeOutput.Value}, ctx.Err()
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
lane.resolved.NormalizeExecutionClass = contracts.ExecutionClassLLMBacked
|
|
|
|
|
lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) {
|
|
|
|
|
return produce(ctx, request.Operation)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for range 2 {
|
|
|
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: store, RawInput: []byte("input"), Workers: 1})
|
2026-11-22 02:34:40 +00:00
|
|
|
if err != nil || len(output.normalizeOutputs) != 2 {
|
|
|
|
|
t.Fatalf("cached %s execution: %v, outputs %d", role, err, len(output.normalizeOutputs))
|
2026-11-22 02:10:05 +00:00
|
|
|
}
|
2026-11-22 02:34:40 +00:00
|
|
|
value, err := notesCodec().Decode(output.normalizeOutputs[0].Artifact.Content)
|
2026-11-22 02:10:05 +00:00
|
|
|
if err != nil || !reflect.DeepEqual(value.Items, []string{"applied"}) {
|
|
|
|
|
t.Fatalf("applied artifact changed on reuse: %#v, %v", value, err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if calls != 1 || store.reads != 2 || store.writes != 1 {
|
|
|
|
|
t.Fatalf("%s cache calls: provider %d, reads/writes %d/%d", role, calls, store.reads, store.writes)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|