package pipeline import ( "context" "encoding/json" "errors" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) func (*nativeEvidence) Enabled() bool { return true } func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) { evidence := &nativeEvidence{} text := strings.Repeat("sk-ordinary-source-content ", 500) reference := strings.Repeat("standalone-reference-content ", 500) doc := typedTestDocument() doc.Units[0].Text = text incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false) if err != nil { t.Fatal(err) } target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"), chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref}, references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}}, } binding := Binding("producer") binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"} if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil { t.Fatal(err) } content := string(evidence.files["inputs.json"]) for _, value := range []string{text, reference, "chunk material", "incoming-value"} { if !strings.Contains(content, value) { t.Fatalf("missing self-contained input %q", value) } } if strings.Contains(content, "configured-option-must-use-native-inspection") { t.Fatal("runtime configuration bypassed secret-aware inspection") } } func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) { _, store := filesystemProducerCache(t) calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})} prepared := prepareRunnerProfile(t, catalog, profile, engine) for _, reuse := range []bool{false, true} { evidence := &nativeEvidence{} var group promptkit.ExecutionGroup output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1}) if err != nil || output.FallbackCount != 0 || calls != 1 { t.Fatalf("run: %#v %v, calls=%d", output, err, calls) } if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 { t.Fatal("missing chunk/extract/merge/normalization boundaries") } if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] { t.Fatal("cache reuse fabricated a native execution") } for name, content := range evidence.files { if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") { continue } var outcome struct { Artifact struct{ Content debugBinaryEnvelope } } if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) { t.Fatalf("missing codec artifact for %s: %s %v", name, content, err) } } if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 { t.Fatal("reused artifact bytes not retained") } var snapshot promptkit.ExecutionGroupSummary if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil { t.Fatal(err) } if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete { t.Fatalf("incorrect current-run totals: %#v", snapshot) } if len(evidence.files["composition.json"]) == 0 { t.Fatal("resolved composition unavailable") } } } func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) { for _, fail := range []bool{false, true} { catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) probe := newRunnerProbe() primary := errors.New("deterministic operation failed") producer := runnerNotesProducer{name: "notes", probe: probe} if fail { producer.failure = primary } lane := registerRunnerNotes(t, catalog, producer) permissive := contracts.ModePermissive lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe) profile := typedResolutionProfile() profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane} evidence := &nativeEvidence{} output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1}) if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) { t.Fatalf("result=%#v %v", output, err) } found := false for name, content := range evidence.files { if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") { continue } found = true var result struct { Outcome contracts.ProducerOutcome Error string Artifact any } if err := json.Unmarshal(content, &result); err != nil { t.Fatal(err) } if fail { if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil { t.Fatalf("fabricated failed artifact: %s", content) } } else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil { t.Fatalf("fallback lost qualification: %s", content) } } if !found { t.Fatal("failed/fallback operation not captured") } } } func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) { primary := errors.New("interpretation evidence failed") engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) fallbacks := 0 operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment} _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) { return nil, nil, &contracts.CaptureError{Cause: primary} }, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" { t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks) } } func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) { binding := Binding("configured-module") binding.Options = map[string]any{"token": "configuration-secret"} validator := Binding("configured-validator") validator.Options = map[string]any{"token": "validator-configuration-secret"} binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}} resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}} content, err := json.Marshal(captureComposition(resolved)) if err != nil { t.Fatal(err) } if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") { t.Fatalf("composition lost identities or bypassed native inspection: %s", content) } if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" { t.Fatal("capture mutated configured module options") } }