Files
notarius/internal/framework/pipeline/operation_capture_test.go

178 lines
8.2 KiB
Go
Raw Normal View History

package pipeline
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
func (*nativeEvidence) Enabled() bool { return true }
func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) {
evidence := &nativeEvidence{}
text := strings.Repeat("sk-ordinary-source-content ", 500)
reference := strings.Repeat("standalone-reference-content ", 500)
doc := typedTestDocument()
doc.Units[0].Text = text
incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false)
if err != nil {
t.Fatal(err)
}
target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"),
chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref},
references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}},
}
binding := Binding("producer")
binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"}
if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil {
t.Fatal(err)
}
content := string(evidence.files["inputs.json"])
for _, value := range []string{text, reference, "chunk material", "incoming-value"} {
if !strings.Contains(content, value) {
t.Fatalf("missing self-contained input %q", value)
}
}
if strings.Contains(content, "configured-option-must-use-native-inspection") {
t.Fatal("runtime configuration bypassed secret-aware inspection")
}
}
func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
for _, reuse := range []bool{false, true} {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1})
if err != nil || output.FallbackCount != 0 || calls != 1 {
t.Fatalf("run: %#v %v, calls=%d", output, err, calls)
}
if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 {
t.Fatal("missing chunk/extract/merge/normalization boundaries")
}
if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] {
t.Fatal("cache reuse fabricated a native execution")
}
for name, content := range evidence.files {
if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") {
continue
}
var outcome struct {
Artifact struct{ Content debugBinaryEnvelope }
}
if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) {
t.Fatalf("missing codec artifact for %s: %s %v", name, content, err)
}
}
if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 {
t.Fatal("reused artifact bytes not retained")
}
var snapshot promptkit.ExecutionGroupSummary
if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil {
t.Fatal(err)
}
if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete {
t.Fatalf("incorrect current-run totals: %#v", snapshot)
}
if len(evidence.files["composition.json"]) == 0 {
t.Fatal("resolved composition unavailable")
}
}
}
func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) {
for _, fail := range []bool{false, true} {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
probe := newRunnerProbe()
primary := errors.New("deterministic operation failed")
producer := runnerNotesProducer{name: "notes", probe: probe}
if fail {
producer.failure = primary
}
lane := registerRunnerNotes(t, catalog, producer)
permissive := contracts.ModePermissive
lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe)
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
evidence := &nativeEvidence{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1})
if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) {
t.Fatalf("result=%#v %v", output, err)
}
found := false
for name, content := range evidence.files {
if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") {
continue
}
found = true
var result struct {
Outcome contracts.ProducerOutcome
Error string
Artifact any
}
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
if fail {
if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil {
t.Fatalf("fabricated failed artifact: %s", content)
}
} else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil {
t.Fatalf("fallback lost qualification: %s", content)
}
}
if !found {
t.Fatal("failed/fallback operation not captured")
}
}
}
func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) {
primary := errors.New("interpretation evidence failed")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
fallbacks := 0
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) {
return nil, nil, &contracts.CaptureError{Cause: primary}
}, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks)
}
}
func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) {
binding := Binding("configured-module")
binding.Options = map[string]any{"token": "configuration-secret"}
validator := Binding("configured-validator")
validator.Options = map[string]any{"token": "validator-configuration-secret"}
binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}}
resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}}
content, err := json.Marshal(captureComposition(resolved))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") {
t.Fatalf("composition lost identities or bypassed native inspection: %s", content)
}
if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" {
t.Fatal("capture mutated configured module options")
}
}