package pipeline import ( "context" "errors" "fmt" "reflect" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) type runnerProbe struct { mu sync.Mutex calls map[string]int eligible map[string][]bool } func newRunnerProbe() *runnerProbe { return &runnerProbe{calls: map[string]int{}, eligible: map[string][]bool{}} } func (p *runnerProbe) record(key string, eligible bool) { p.mu.Lock() defer p.mu.Unlock() p.calls[key]++ p.eligible[key] = append(p.eligible[key], eligible) } type runnerNotesProducer struct { name string probe *runnerProbe engine *promptkit.Engine native bool references bool failure error cancel context.CancelFunc } func (p *runnerNotesProducer) Key() string { return p.name } func (p *runnerNotesProducer) ReferenceSlots() []contracts.ReferenceSlot { if !p.references { return nil } return []contracts.ReferenceSlot{{Name: "context", AcceptedArtifactKinds: []contracts.ArtifactKind{"test/notes"}, AcceptedMediaTypes: []string{"application/json"}}} } func (p *runnerNotesProducer) fallback(ctx context.Context, role string) (codecNotes, error) { p.probe.record(p.name+"/"+role+"/fallback", false) return codecNotes{Items: []string{}}, ctx.Err() } func (p *runnerNotesProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) { p.probe.record(p.name+"/extract", request.Operation.ReuseEligible) if p.cancel != nil { p.cancel() } if p.failure != nil { return contracts.TypedExtractionResult[codecNotes]{}, p.failure } if p.native { initial := nativeProducerInitial() initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID value, outcome, err := RunProducer(ctx, p.engine, request.Operation, initial, func(ctx context.Context, result *promptkit.RunResult) (*codecNotes, []string, error) { number, _, err := interpretProducerNumber(ctx, result) if err != nil { return nil, nil, err } return &codecNotes{Items: []string{fmt.Sprint(*number)}}, nil, nil }, producerCorrection, func(ctx context.Context, _ *codecNotes) (codecNotes, error) { return p.fallback(ctx, "extract") }) return contracts.TypedExtractionResult[codecNotes]{Value: value, Outcome: outcome}, err } return contracts.TypedExtractionResult[codecNotes]{Value: codecNotes{Items: []string{p.name}}, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "extract") }}, nil } func (p *runnerNotesProducer) Merge(ctx context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) { p.probe.record(p.name+"/merge", request.Operation.ReuseEligible) if p.references { if slot, ok := request.References.Slots["context"]; ok && len(slot.Items) > 0 { p.probe.record(p.name+"/reference", request.Operation.ReuseEligible) } } value := codecNotes{Items: []string{}} for _, output := range request.ExtractOutputs { value.Items = append(value.Items, output.Value.Items...) } return contracts.TypedMergeResult[codecNotes]{Value: value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "merge") }}, nil } func (p *runnerNotesProducer) Normalize(ctx context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) { p.probe.record(p.name+"/normalize", request.Operation.ReuseEligible) return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "normalize") }}, nil } func registerRunnerNotes(t *testing.T, catalog ModuleCatalog, producer runnerNotesProducer) ArtifactLaneProfile { t.Helper() class := contracts.ExecutionClassDeterministic if producer.native { class = contracts.ExecutionClassLLMBacked } spec := func(stage ModuleStage) ModuleSpec { return ModuleSpec{Key: producer.name, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes", ReferenceSlots: producer.ReferenceSlots()} } extractSpec := spec(StageExtract) extractSpec.ExecutionClass = class if err := RegisterExtractorBuilder(catalog.Extractors, extractSpec, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Extractor[codecNotes], error) { copy := producer copy.engine = request.Dependencies.Engine return ©, nil }); err != nil { t.Fatal(err) } if err := RegisterMerger(catalog.Mergers, spec(StageMerge), func() (contracts.Merger[codecNotes], error) { copy := producer; return ©, nil }); err != nil { t.Fatal(err) } if err := RegisterNormalizer(catalog.Normalizers, spec(StageNormalize), func() (contracts.Normalizer[codecNotes], error) { copy := producer; return ©, nil }); err != nil { t.Fatal(err) } return ArtifactLaneProfile{Extract: Binding(producer.name), Merge: Binding(producer.name), Normalize: Binding(producer.name)} } func prepareRunnerProfile(t *testing.T, catalog ModuleCatalog, profile PipelineProfile, engine *promptkit.Engine) *PreparedPipeline { t.Helper() resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) if err != nil { t.Fatal(err) } resolved, _, err = MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{}) if err != nil { t.Fatal(err) } prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{Engine: engine}) if err != nil { t.Fatal(err) } return prepared } type namedRunnerValidator struct { name string notesAssessmentValidator } func (v namedRunnerValidator) Name() string { return v.name } func registerRunnerRejection(t *testing.T, catalog ModuleCatalog, probe *runnerProbe) ValidatorOverride { t.Helper() if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: "reject", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return namedRunnerValidator{"reject", notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { probe.record(request.ModuleKey+"/"+request.Stage+"/validator", false) return contracts.ValidationResult{ReasonCode: "domain_rejection", CorrectionGuidance: "Return a supported value."}, nil })}, nil }); err != nil { t.Fatal(err) } return ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("reject")}} } func TestRunnerPropagatesLaneFallbackWithoutCountingDependents(t *testing.T) { for _, stage := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { t.Run(string(stage), func(t *testing.T) { probe := newRunnerProbe() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) affected := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "affected", probe: probe}) unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "unrelated", probe: probe}) reject := registerRunnerRejection(t, catalog, probe) permissive := contracts.ModePermissive switch stage { case StageExtract: affected.Extract.Validators, affected.Extract.Mode = reject, &permissive case StageMerge: affected.Merge.Validators, affected.Merge.Mode = reject, &permissive case StageNormalize: affected.Normalize.Validators, affected.Normalize.Mode = reject, &permissive } profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"affected": affected, "unrelated": unrelated} output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1}) if err != nil { t.Fatal(err) } if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 2 { t.Fatalf("fallback count=%d, outputs=%d", output.FallbackCount, len(output.NormalizeOutputs)) } after := false for _, role := range []ModuleStage{StageExtract, StageMerge, StageNormalize} { for _, name := range []string{"affected", "unrelated"} { key := name + "/" + string(role) want := name != "affected" || !after if probe.calls[key] != 1 || !reflect.DeepEqual(probe.eligible[key], []bool{want}) { t.Fatalf("%s calls=%d eligibility=%#v, want %t", key, probe.calls[key], probe.eligible[key], want) } } if role == stage { after = true } } if probe.calls["affected/"+string(stage)+"/validator"] != 1 || probe.calls["affected/"+string(stage)+"/fallback"] != 1 { t.Fatalf("deterministic work was retried or fallback revalidated: %#v", probe.calls) } lanes := output.normalizeReuseEligibility for key, eligible := range lanes { if eligible != (key.laneID == "unrelated") { t.Fatalf("normalized eligibility = %#v", lanes) } } }) } } func TestRunnerAssessesNativeRevisionsWithoutOuterRevalidation(t *testing.T) { probe := newRunnerProbe() providerCalls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if request.Target.Model != "runner-model" || request.Prompt.SessionID != "runner-session" { t.Fatalf("run-wide override missing: %#v", request) } value := providerCalls providerCalls++ return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil })) catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: probe, native: true}) for _, name := range []string{"first", "second"} { if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) { return namedRunnerValidator{name, notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { probe.record(name+"/"+request.Value.Items[0], true) if name == "second" && request.Value.Items[0] == "0" { return contracts.ValidationResult{ReasonCode: "not_one", CorrectionGuidance: "Return value one."}, nil } return contracts.ValidationResult{Approved: true}, nil })}, nil }); err != nil { t.Fatal(err) } } lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first"), Binding("second")}} profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"native": lane} profile.LLMProfile = "chosen" turns := 1 profile.MaxAdditionalTurns = &turns output, err := New().Run(context.Background(), RunInput{ Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), SessionID: "runner-session", Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: "runner-model"}}, }) if err != nil { t.Fatal(err) } if providerCalls != 2 || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 1 { t.Fatalf("native calls=%d, output=%#v", providerCalls, output) } for _, key := range []string{"first/0", "second/0", "first/1", "second/1", "native/extract", "native/merge", "native/normalize"} { if probe.calls[key] != 1 { t.Fatalf("%s called %d times", key, probe.calls[key]) } } } func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers(t *testing.T) { probe := newRunnerProbe() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "producer", probe: probe}) earlier := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "earlier", probe: probe}) consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe, references: true}) unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "later", probe: probe}) producer.Normalize.Validators = registerRunnerRejection(t, catalog, probe) permissive := contracts.ModePermissive producer.Normalize.Mode = &permissive consumer.Merge.References = map[string]ReferenceSource{"context": GeneratedReference("build", "producer")} profile := typedResolutionProfile() profile.Artifacts = nil profile.Steps = []PipelineStepProfile{ {ID: "build", Artifacts: map[string]ArtifactLaneProfile{"producer": producer, "earlier": earlier}}, {ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer, "later": unrelated}}, } output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1}) if err != nil { t.Fatal(err) } if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 4 || probe.calls["consumer/reference"] != 1 { t.Fatalf("handoff failed: count=%d outputs=%d observations=%#v", output.FallbackCount, len(output.NormalizeOutputs), probe.calls) } for _, key := range []string{"earlier/extract", "earlier/merge", "earlier/normalize", "later/extract", "later/merge", "later/normalize", "consumer/extract"} { if !reflect.DeepEqual(probe.eligible[key], []bool{true}) { t.Fatalf("unaffected work %s tainted: %#v", key, probe.eligible[key]) } } for _, key := range []string{"consumer/merge", "consumer/normalize"} { if !reflect.DeepEqual(probe.eligible[key], []bool{false}) { t.Fatalf("dependent %s remained reusable: %#v", key, probe.eligible[key]) } } if output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "consumer")] || !output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "later")] { t.Fatalf("generated lineage = %#v", output.normalizeReuseEligibility) } } type runnerFallbackChunker struct{} func (runnerFallbackChunker) Key() string { return "fallback-chunk" } func (runnerFallbackChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil } func (runnerFallbackChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { return contracts.ChunkPlanResult{Plan: typedTestPlan(request.Source), Outcome: contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: "selected test fallback", ReuseEligible: false}}, nil } func TestRunnerChunkFallbackTaintsEveryDerivedOperation(t *testing.T) { probe := newRunnerProbe() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil { t.Fatal(err) } lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe}) profile := typedResolutionProfile() profile.Chunk = Binding("fallback-chunk") permissive := contracts.ModePermissive profile.Chunk.Mode = &permissive profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane} output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")}) if err != nil { t.Fatal(err) } if output.FallbackCount != 1 || len(output.NormalizeOutputs) != 1 { t.Fatalf("chunk fallback = %#v", output) } for _, role := range []string{"extract", "merge", "normalize"} { if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{false}) { t.Fatalf("chunk fallback did not taint %s: %#v", role, probe.eligible) } } } func TestRunnerDoesNotRetryOrFallbackDeterministicOperationErrors(t *testing.T) { probe := newRunnerProbe() failure := errors.New("deterministic operation failed") catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, failure: failure}) permissive := contracts.ModePermissive profile := typedResolutionProfile() profile.Mode = &permissive profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane} _, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")}) if !errors.Is(err, failure) || probe.calls["notes/extract"] != 1 || probe.calls["notes/extract/fallback"] != 0 || probe.calls["notes/merge"] != 0 { t.Fatalf("operation failure = %v, calls %#v", err, probe.calls) } } func TestRunnerDoesNotAcceptModuleFallbackUnderStrictControls(t *testing.T) { probe := newRunnerProbe() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil { t.Fatal(err) } profile := typedResolutionProfile() profile.Chunk = Binding("fallback-chunk") profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})} output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")}) if err == nil || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 || probe.calls["notes/extract"] != 0 { t.Fatalf("strict controls accepted module fallback: %#v %v", output, err) } } func TestRunnerDiscardsDeterministicOutputReturnedAfterCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() probe := newRunnerProbe() catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, cancel: cancel}) profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane} output, err := New().Run(ctx, RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")}) if !errors.Is(err, context.Canceled) || output.FallbackCount != 0 || len(output.NormalizeOutputs) != 0 || probe.calls["notes/merge"] != 0 || probe.calls["notes/extract/fallback"] != 0 { t.Fatalf("cancellation = %#v %v, calls %#v", output, err, probe.calls) } } var _ contracts.Chunker = runnerFallbackChunker{} var _ contracts.Extractor[codecNotes] = (*runnerNotesProducer)(nil) var _ contracts.Merger[codecNotes] = (*runnerNotesProducer)(nil) var _ contracts.Normalizer[codecNotes] = (*runnerNotesProducer)(nil)