Files
notarius/internal/framework/pipeline/runner_operation_test.go

379 lines
18 KiB
Go

package pipeline
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type runnerProbe struct {
mu sync.Mutex
calls map[string]int
eligible map[string][]bool
}
func newRunnerProbe() *runnerProbe {
return &runnerProbe{calls: map[string]int{}, eligible: map[string][]bool{}}
}
func (p *runnerProbe) record(key string, eligible bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.calls[key]++
p.eligible[key] = append(p.eligible[key], eligible)
}
type runnerNotesProducer struct {
name string
probe *runnerProbe
engine *promptkit.Engine
native bool
references bool
failure error
cancel context.CancelFunc
}
func (p *runnerNotesProducer) Key() string { return p.name }
func (p *runnerNotesProducer) ReferenceSlots() []contracts.ReferenceSlot {
if !p.references {
return nil
}
return []contracts.ReferenceSlot{{Name: "context", AcceptedArtifactKinds: []contracts.ArtifactKind{"test/notes"}, AcceptedMediaTypes: []string{"application/json"}}}
}
func (p *runnerNotesProducer) fallback(ctx context.Context, role string) (codecNotes, error) {
p.probe.record(p.name+"/"+role+"/fallback", false)
return codecNotes{Items: []string{}}, ctx.Err()
}
func (p *runnerNotesProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[codecNotes]) (contracts.TypedExtractionResult[codecNotes], error) {
p.probe.record(p.name+"/extract", request.Operation.ReuseEligible)
if p.cancel != nil {
p.cancel()
}
if p.failure != nil {
return contracts.TypedExtractionResult[codecNotes]{}, p.failure
}
if p.native {
initial := nativeProducerInitial()
initial.ProfileID, initial.SessionID = request.LLMProfile, request.SessionID
value, outcome, err := RunProducer(ctx, p.engine, request.Operation, initial,
func(ctx context.Context, result *promptkit.RunResult) (*codecNotes, []string, error) {
number, _, err := interpretProducerNumber(ctx, result)
if err != nil {
return nil, nil, err
}
return &codecNotes{Items: []string{fmt.Sprint(*number)}}, nil, nil
}, producerCorrection, func(ctx context.Context, _ *codecNotes) (codecNotes, error) { return p.fallback(ctx, "extract") })
return contracts.TypedExtractionResult[codecNotes]{Value: value, Outcome: outcome}, err
}
return contracts.TypedExtractionResult[codecNotes]{Value: codecNotes{Items: []string{p.name}}, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "extract") }}, nil
}
func (p *runnerNotesProducer) Merge(ctx context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
p.probe.record(p.name+"/merge", request.Operation.ReuseEligible)
if p.references {
if slot, ok := request.References.Slots["context"]; ok && len(slot.Items) > 0 {
p.probe.record(p.name+"/reference", request.Operation.ReuseEligible)
}
}
value := codecNotes{Items: []string{}}
for _, output := range request.ExtractOutputs {
value.Items = append(value.Items, output.Value.Items...)
}
return contracts.TypedMergeResult[codecNotes]{Value: value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "merge") }}, nil
}
func (p *runnerNotesProducer) Normalize(ctx context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
p.probe.record(p.name+"/normalize", request.Operation.ReuseEligible)
return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value, Fallback: func(ctx context.Context) (codecNotes, error) { return p.fallback(ctx, "normalize") }}, nil
}
func registerRunnerNotes(t *testing.T, catalog ModuleCatalog, producer runnerNotesProducer) ArtifactLaneProfile {
t.Helper()
class := contracts.ExecutionClassDeterministic
if producer.native {
class = contracts.ExecutionClassLLMBacked
}
spec := func(stage ModuleStage) ModuleSpec {
return ModuleSpec{Key: producer.name, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic, ArtifactKind: "test/notes", ReferenceSlots: producer.ReferenceSlots()}
}
extractSpec := spec(StageExtract)
extractSpec.ExecutionClass = class
if err := RegisterExtractorBuilder(catalog.Extractors, extractSpec, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Extractor[codecNotes], error) {
copy := producer
copy.engine = request.Dependencies.Engine
return &copy, nil
}); err != nil {
t.Fatal(err)
}
if err := RegisterMerger(catalog.Mergers, spec(StageMerge), func() (contracts.Merger[codecNotes], error) { copy := producer; return &copy, nil }); err != nil {
t.Fatal(err)
}
if err := RegisterNormalizer(catalog.Normalizers, spec(StageNormalize), func() (contracts.Normalizer[codecNotes], error) { copy := producer; return &copy, nil }); err != nil {
t.Fatal(err)
}
return ArtifactLaneProfile{Extract: Binding(producer.name), Merge: Binding(producer.name), Normalize: Binding(producer.name)}
}
func prepareRunnerProfile(t *testing.T, catalog ModuleCatalog, profile PipelineProfile, engine *promptkit.Engine) *PreparedPipeline {
t.Helper()
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatal(err)
}
resolved, _, err = MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{})
if err != nil {
t.Fatal(err)
}
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{Engine: engine})
if err != nil {
t.Fatal(err)
}
return prepared
}
type namedRunnerValidator struct {
name string
notesAssessmentValidator
}
func (v namedRunnerValidator) Name() string { return v.name }
func registerRunnerRejection(t *testing.T, catalog ModuleCatalog, probe *runnerProbe) ValidatorOverride {
t.Helper()
if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: "reject", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
return namedRunnerValidator{"reject", notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
probe.record(request.ModuleKey+"/"+request.Stage+"/validator", false)
return contracts.ValidationResult{ReasonCode: "domain_rejection", CorrectionGuidance: "Return a supported value."}, nil
})}, nil
}); err != nil {
t.Fatal(err)
}
return ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("reject")}}
}
func TestRunnerPropagatesLaneFallbackWithoutCountingDependents(t *testing.T) {
for _, stage := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
t.Run(string(stage), func(t *testing.T) {
probe := newRunnerProbe()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
affected := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "affected", probe: probe})
unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "unrelated", probe: probe})
reject := registerRunnerRejection(t, catalog, probe)
permissive := contracts.ModePermissive
switch stage {
case StageExtract:
affected.Extract.Validators, affected.Extract.Mode = reject, &permissive
case StageMerge:
affected.Merge.Validators, affected.Merge.Mode = reject, &permissive
case StageNormalize:
affected.Normalize.Validators, affected.Normalize.Mode = reject, &permissive
}
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"affected": affected, "unrelated": unrelated}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
if err != nil {
t.Fatal(err)
}
if output.FallbackCount != 1 || len(output.normalizeOutputs) != 2 {
t.Fatalf("fallback count=%d, outputs=%d", output.FallbackCount, len(output.normalizeOutputs))
}
after := false
for _, role := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
for _, name := range []string{"affected", "unrelated"} {
key := name + "/" + string(role)
want := name != "affected" || !after
if probe.calls[key] != 1 || !reflect.DeepEqual(probe.eligible[key], []bool{want}) {
t.Fatalf("%s calls=%d eligibility=%#v, want %t", key, probe.calls[key], probe.eligible[key], want)
}
}
if role == stage {
after = true
}
}
if probe.calls["affected/"+string(stage)+"/validator"] != 1 || probe.calls["affected/"+string(stage)+"/fallback"] != 1 {
t.Fatalf("deterministic work was retried or fallback revalidated: %#v", probe.calls)
}
lanes := output.normalizeReuseEligibility
for key, eligible := range lanes {
if eligible != (key.laneID == "unrelated") {
t.Fatalf("normalized eligibility = %#v", lanes)
}
}
})
}
}
func TestRunnerAssessesNativeRevisionsWithoutOuterRevalidation(t *testing.T) {
probe := newRunnerProbe()
providerCalls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.Model != "runner-model" || request.Prompt.SessionID != "runner-session" {
t.Fatalf("run-wide override missing: %#v", request)
}
value := providerCalls
providerCalls++
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: probe, native: true})
for _, name := range []string{"first", "second"} {
if err := RegisterTypedValidator(catalog.Validators, "test/notes", ValidatorSpec{Key: name, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.TypedValidator[codecNotes], error) {
return namedRunnerValidator{name, notesAssessmentValidator(func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
probe.record(name+"/"+request.Value.Items[0], true)
if name == "second" && request.Value.Items[0] == "0" {
return contracts.ValidationResult{ReasonCode: "not_one", CorrectionGuidance: "Return value one."}, nil
}
return contracts.ValidationResult{Approved: true}, nil
})}, nil
}); err != nil {
t.Fatal(err)
}
}
lane.Extract.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{Binding("first"), Binding("second")}}
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"native": lane}
profile.LLMProfile = "chosen"
turns := 1
profile.MaxAdditionalTurns = &turns
output, err := New().Run(context.Background(), RunInput{
Prepared: prepareRunnerProfile(t, catalog, profile, engine), RawInput: []byte("input"), SessionID: "runner-session",
Execution: contracts.ExecutionOptions{Execution: &promptkit.ExecutionTargetOverride{Model: "runner-model"}},
})
if err != nil {
t.Fatal(err)
}
if providerCalls != 2 || output.FallbackCount != 0 || len(output.normalizeOutputs) != 1 {
t.Fatalf("native calls=%d, output=%#v", providerCalls, output)
}
for _, key := range []string{"first/0", "second/0", "first/1", "second/1", "native/extract", "native/merge", "native/normalize"} {
if probe.calls[key] != 1 {
t.Fatalf("%s called %d times", key, probe.calls[key])
}
}
}
func TestRunnerGeneratedFallbackReferencesTaintOnlyActualConsumers(t *testing.T) {
probe := newRunnerProbe()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
producer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "producer", probe: probe})
earlier := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "earlier", probe: probe})
consumer := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "consumer", probe: probe, references: true})
unrelated := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "later", probe: probe})
producer.Normalize.Validators = registerRunnerRejection(t, catalog, probe)
permissive := contracts.ModePermissive
producer.Normalize.Mode = &permissive
consumer.Merge.References = map[string]ReferenceSource{"context": GeneratedReference("build", "producer")}
profile := typedResolutionProfile()
profile.Artifacts = nil
profile.Steps = []PipelineStepProfile{
{ID: "build", Artifacts: map[string]ArtifactLaneProfile{"producer": producer, "earlier": earlier}},
{ID: "consume", Artifacts: map[string]ArtifactLaneProfile{"consumer": consumer, "later": unrelated}},
}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Workers: 1})
if err != nil {
t.Fatal(err)
}
if output.FallbackCount != 1 || len(output.normalizeOutputs) != 4 || probe.calls["consumer/reference"] != 1 {
t.Fatalf("handoff failed: count=%d outputs=%d observations=%#v", output.FallbackCount, len(output.normalizeOutputs), probe.calls)
}
for _, key := range []string{"earlier/extract", "earlier/merge", "earlier/normalize", "later/extract", "later/merge", "later/normalize", "consumer/extract"} {
if !reflect.DeepEqual(probe.eligible[key], []bool{true}) {
t.Fatalf("unaffected work %s tainted: %#v", key, probe.eligible[key])
}
}
for _, key := range []string{"consumer/merge", "consumer/normalize"} {
if !reflect.DeepEqual(probe.eligible[key], []bool{false}) {
t.Fatalf("dependent %s remained reusable: %#v", key, probe.eligible[key])
}
}
if output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "consumer")] || !output.normalizeReuseEligibility[generatedOutputKeyFor("consume", "later")] {
t.Fatalf("generated lineage = %#v", output.normalizeReuseEligibility)
}
}
type runnerFallbackChunker struct{}
func (runnerFallbackChunker) Key() string { return "fallback-chunk" }
func (runnerFallbackChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (runnerFallbackChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{Plan: typedTestPlan(request.Source), Outcome: contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: "selected test fallback", ReuseEligible: false}}, nil
}
func TestRunnerChunkFallbackTaintsEveryDerivedOperation(t *testing.T) {
probe := newRunnerProbe()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil {
t.Fatal(err)
}
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})
profile := typedResolutionProfile()
profile.Chunk = Binding("fallback-chunk")
permissive := contracts.ModePermissive
profile.Chunk.Mode = &permissive
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
if err != nil {
t.Fatal(err)
}
if output.FallbackCount != 1 || len(output.normalizeOutputs) != 1 {
t.Fatalf("chunk fallback = %#v", output)
}
for _, role := range []string{"extract", "merge", "normalize"} {
if !reflect.DeepEqual(probe.eligible["notes/"+role], []bool{false}) {
t.Fatalf("chunk fallback did not taint %s: %#v", role, probe.eligible)
}
}
}
func TestRunnerDoesNotRetryOrFallbackDeterministicOperationErrors(t *testing.T) {
probe := newRunnerProbe()
failure := errors.New("deterministic operation failed")
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, failure: failure})
permissive := contracts.ModePermissive
profile := typedResolutionProfile()
profile.Mode = &permissive
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
_, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
if !errors.Is(err, failure) || probe.calls["notes/extract"] != 1 || probe.calls["notes/extract/fallback"] != 0 || probe.calls["notes/merge"] != 0 {
t.Fatalf("operation failure = %v, calls %#v", err, probe.calls)
}
}
func TestRunnerDoesNotAcceptModuleFallbackUnderStrictControls(t *testing.T) {
probe := newRunnerProbe()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "fallback-chunk", Stage: StageChunk, ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.Chunker, error) { return runnerFallbackChunker{}, nil }); err != nil {
t.Fatal(err)
}
profile := typedResolutionProfile()
profile.Chunk = Binding("fallback-chunk")
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe})}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
if err == nil || output.FallbackCount != 0 || len(output.normalizeOutputs) != 0 || probe.calls["notes/extract"] != 0 {
t.Fatalf("strict controls accepted module fallback: %#v %v", output, err)
}
}
func TestRunnerDiscardsDeterministicOutputReturnedAfterCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
probe := newRunnerProbe()
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
lane := registerRunnerNotes(t, catalog, runnerNotesProducer{name: "notes", probe: probe, cancel: cancel})
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
output, err := New().Run(ctx, RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input")})
if !errors.Is(err, context.Canceled) || output.FallbackCount != 0 || len(output.normalizeOutputs) != 0 || probe.calls["notes/merge"] != 0 || probe.calls["notes/extract/fallback"] != 0 {
t.Fatalf("cancellation = %#v %v, calls %#v", output, err, probe.calls)
}
}
var _ contracts.Chunker = runnerFallbackChunker{}
var _ contracts.Extractor[codecNotes] = (*runnerNotesProducer)(nil)
var _ contracts.Merger[codecNotes] = (*runnerNotesProducer)(nil)
var _ contracts.Normalizer[codecNotes] = (*runnerNotesProducer)(nil)