package pipeline import ( "context" "encoding/json" "errors" "reflect" "strings" "sync/atomic" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) func TestRunnerChecksCurrentSourceOnConcurrentCacheHits(t *testing.T) { _, cache := filesystemProducerCache(t) var calls atomic.Int32 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls.Add(1) return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) const chunkCount = 4 prepared := preparedConcurrentPipeline(t, chunkCount) for i := range prepared.Steps[0].lanes { lane := &prepared.Steps[0].lanes[i] if lane.resolved.ArtifactKind != "test/notes" { continue } lane.resolved.ExtractExecutionClass = contracts.ExecutionClassLLMBacked lane.typed.cacheEvidence = func(any) ([]source.SourceRef, error) { return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, nil } lane.typed.extract = func(ctx context.Context, _ any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { initial := nativeProducerInitial() initial.Inputs["source"] = promptkit.Inline(string(request.Chunk.Content)) value, outcome, err := RunProducer(ctx, engine, request.Operation, initial, func(context.Context, *promptkit.RunResult) (*any, []string, error) { var value any = codecNotes{Items: []string{"accepted"}} return &value, nil, nil }, nil, nil) return erasedTypedResult{Value: value, Outcome: outcome}, err } } for run := 0; run < 2; run++ { // The filesystem store is concurrency-safe; its counting test wrapper is not. output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: cache.inner, RawInput: []byte("input"), Workers: chunkCount}) if err != nil || len(output.normalizeOutputs) != 2 { t.Fatalf("run %d: outputs=%d error=%v", run, len(output.normalizeOutputs), err) } } if calls.Load() != chunkCount { t.Fatalf("cache hits executed model work: calls=%d", calls.Load()) } } func TestRunnerReusesAcceptedPredecessorAfterDownstreamFailure(t *testing.T) { _, store := filesystemProducerCache(t) probe := newRunnerProbe() calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "cached-producer", probe: probe, native: true}) consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe}) profile := typedResolutionProfile() profile.Artifacts = nil profile.Steps = []PipelineStepProfile{ {ID: "produce", Artifacts: map[string]ArtifactLaneProfile{"producer": producer}}, {ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer}}, } prepared := prepareRunnerProfile(t, catalog, profile, engine) failure := errors.New("downstream application failure") first := true consumerOperation := prepared.Steps[1].lanes[0].typed.extract prepared.Steps[1].lanes[0].typed.extract = func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) { if first { return erasedTypedResult{}, failure } return consumerOperation(ctx, implementation, request) } var observations []contracts.CacheObservation input := RunInput{Prepared: prepared, RawInput: []byte("input"), Workers: 1, Cache: store, ObserveCache: func(_ context.Context, origin contracts.DiagnosticOrigin, observation contracts.CacheObservation) error { if origin.ModuleKey != "cached-producer" || origin.Stage != contracts.DiagnosticOriginStageExtract || observation.ChunkID == "" || observation.ChunkIndex == nil { t.Fatalf("lost cache scope: %#v %#v", origin, observation) } observations = append(observations, observation) return nil }, } if _, err := New().Run(context.Background(), input); !errors.Is(err, failure) { t.Fatalf("downstream failure=%v", err) } if calls != 1 || store.reads != 1 || store.writes != 1 { t.Fatalf("predecessor not stored before downstream failure: calls %d reads/writes %d/%d", calls, store.reads, store.writes) } first = false output, err := New().Run(context.Background(), input) if err != nil || len(output.normalizeOutputs) != 2 || calls != 1 || store.reads != 2 || store.writes != 1 || output.FallbackCount != 0 { t.Fatalf("rerun=%v outputs=%d calls=%d cache=%d/%d", err, len(output.normalizeOutputs), calls, store.reads, store.writes) } var decisions []string for _, observation := range observations { decisions = append(decisions, observation.Decision) } if !reflect.DeepEqual(decisions, []string{"prepared", "miss", "stored", "prepared", "hit"}) { t.Fatalf("cache decisions=%v", decisions) } // The deterministic merge/normalize work runs normally on both invocations; // the reused producer's semantic assessor is not repeated. if probe.calls["cached-producer/merge"] != 2 || probe.calls["cached-producer/normalize"] != 2 { t.Fatalf("deterministic work was incorrectly cached: %v", probe.calls) } } func TestRunnerCacheWarningsDoNotFailSuccessfulPublication(t *testing.T) { _, store := filesystemProducerCache(t) store.readErr = errors.New("cache read unavailable") store.writeErr = errors.New(strings.Repeat("cache write unavailable ", 400)) engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})} output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), Cache: store, Workers: 1}) if err != nil || len(output.normalizeOutputs) != 1 || output.FallbackCount != 0 { t.Fatalf("cache warnings failed successful work: %v, outputs=%d", err, len(output.normalizeOutputs)) } projection, err := contracts.ProjectDiagnosticCollection(output.Diagnostics) if err != nil || projection.WarningOccurrenceCount != 2 { t.Fatalf("cache diagnostics=%#v %v", projection, err) } for _, warning := range projection.Warnings { if warning.Origin.ModuleKey != "native" || warning.Origin.LaneID != "notes" || warning.Disposition != contracts.DiagnosticDispositionWarning { t.Fatalf("unscoped cache warning=%#v", warning) } } } func TestTypedCacheDecodeUsesCodecAndCurrentSourceWithoutSemanticAssessment(t *testing.T) { _, store := filesystemProducerCache(t) doc := typedTestDocument() codec := notesCodec() registry := NewArtifactCodecRegistry() if err := RegisterArtifactCodec(registry, codec); err != nil { t.Fatal(err) } entry, _, err := registry.entry("test/notes") if err != nil { t.Fatal(err) } evidence := NewArtifactEvidenceRegistry() if err := RegisterArtifactEvidence(evidence, "test/notes", func(value codecNotes) []source.SourceRef { if value.Details == nil { return nil } return []source.SourceRef{{SourceID: value.Details.Name, StartUnitID: 1, EndUnitID: 1}} }); err != nil { t.Fatal(err) } projector, _, err := evidence.entry("test/notes") if err != nil { t.Fatal(err) } typed := &preparedTypedLane{codec: entry, cacheEvidence: projector.project} var warnings []contracts.ProducerDiagnostic cache := artifactProducerCache(RunInput{Cache: store}, typed, contracts.ExecutionClassLLMBacked, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, ModuleKey: "native"}, doc, nil, &warnings) specialized := specializeProducerOperation[codecNotes](contracts.ProducerOperation[any]{Cache: cache}).Cache for _, test := range []struct { name string content []byte valid bool }{ {"current source", []byte(`{"items":["accepted"],"details":{"name":"source"}}`), true}, {"stale source", []byte(`{"items":["accepted"],"details":{"name":"old-source"}}`), false}, {"unusable codec", []byte(`{"items":"not a collection"}`), false}, } { t.Run(test.name, func(t *testing.T) { value, err := specialized.Decode(test.content) if (err == nil) != test.valid { t.Fatalf("decoded=%#v %v", value, err) } }) } // A codec mismatch stays a typed error at the registry boundary. wrong := specializeProducerOperation[codecScore](contracts.ProducerOperation[any]{Cache: cache}).Cache if _, err := wrong.Decode([]byte(`{"items":[]}`)); err == nil { t.Fatal("cache decode crossed an incompatible Go type") } } func TestProducerCacheCancellationDuringHitCannotDeliverAcceptance(t *testing.T) { _, store := filesystemProducerCache(t) engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { t.Fatal("cache hit made a provider call") return nil, nil })) ctx, cancel := context.WithCancel(context.Background()) defer cancel() operation := numberCacheOperation(store) operation.Cache.Observe = func(ctx context.Context, observation contracts.CacheObservation) error { if observation.Prepared != nil { return store.inner.Store(ctx, contracts.CacheAuto, observation.MessageContentHash, []byte("1")) } if observation.Decision == "hit" { cancel() } return nil } _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) if !errors.Is(err, context.Canceled) || outcome.Disposition != "" { t.Fatalf("canceled hit delivered acceptance: %#v %v", outcome, err) } } func TestRunnerPreparesCacheEvidenceWithoutEnablingEvidenceOutput(t *testing.T) { catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) catalog.ArtifactEvidence = NewArtifactEvidenceRegistry() if err := RegisterArtifactEvidence(catalog.ArtifactEvidence, "test/notes", func(value codecNotes) []source.SourceRef { return []source.SourceRef{{SourceID: "source", StartUnitID: 1, EndUnitID: 1}} }); err != nil { t.Fatal(err) } prepared := prepareRunnerProfile(t, catalog, typedResolutionProfile(), nil) for _, lane := range prepared.Steps[0].lanes { if lane.resolved.ArtifactKind == "test/notes" { if lane.typed.cacheEvidence == nil { t.Fatal("source usability depended on evidence-context publication") } content, _ := json.Marshal(codecNotes{Items: []string{}}) _, store := filesystemProducerCache(t) var warnings []contracts.ProducerDiagnostic cache := artifactProducerCache(RunInput{Cache: store}, lane.typed, contracts.ExecutionClassLLMBacked, contracts.DiagnosticOrigin{}, typedTestDocument(), nil, &warnings) if _, err := cache.Decode(content); err != nil { t.Fatal(err) } } } } func TestRunnerCachesAppliedMergeAndNormalizationArtifacts(t *testing.T) { for _, role := range []ModuleStage{StageMerge, StageNormalize} { t.Run(string(role), func(t *testing.T) { _, store := filesystemProducerCache(t) calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) prepared := preparedConcurrentPipeline(t, 1) lane := &prepared.Steps[0].lanes[0] produce := func(ctx context.Context, operation contracts.ProducerOperation[any]) (erasedTypedResult, error) { value, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*any, []string, error) { var applied any = codecNotes{Items: []string{"applied"}} return &applied, nil, nil }, nil, nil) return erasedTypedResult{Value: value, Outcome: outcome}, err } if role == StageMerge { lane.resolved.MergeExecutionClass = contracts.ExecutionClassLLMBacked lane.typed.merge = func(ctx context.Context, _ any, request contracts.TypedMergeRequest[any]) (erasedTypedResult, error) { return produce(ctx, request.Operation) } lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { return erasedTypedResult{Value: request.MergeOutput.Value}, ctx.Err() } } else { lane.resolved.NormalizeExecutionClass = contracts.ExecutionClassLLMBacked lane.typed.normalize = func(ctx context.Context, _ any, request contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error) { return produce(ctx, request.Operation) } } for range 2 { output, err := New().Run(context.Background(), RunInput{Prepared: prepared, Cache: store, RawInput: []byte("input"), Workers: 1}) if err != nil || len(output.normalizeOutputs) != 2 { t.Fatalf("cached %s execution: %v, outputs %d", role, err, len(output.normalizeOutputs)) } value, err := notesCodec().Decode(output.normalizeOutputs[0].Artifact.Content) if err != nil || !reflect.DeepEqual(value.Items, []string{"applied"}) { t.Fatalf("applied artifact changed on reuse: %#v, %v", value, err) } } if calls != 1 || store.reads != 2 || store.writes != 1 { t.Fatalf("%s cache calls: provider %d, reads/writes %d/%d", role, calls, store.reads, store.writes) } }) } }