178 lines
8.2 KiB
Go
178 lines
8.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func (*nativeEvidence) Enabled() bool { return true }
|
|
|
|
func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) {
|
|
evidence := &nativeEvidence{}
|
|
text := strings.Repeat("sk-ordinary-source-content ", 500)
|
|
reference := strings.Repeat("standalone-reference-content ", 500)
|
|
doc := typedTestDocument()
|
|
doc.Units[0].Text = text
|
|
incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"),
|
|
chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref},
|
|
references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}},
|
|
}
|
|
binding := Binding("producer")
|
|
binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"}
|
|
if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
content := string(evidence.files["inputs.json"])
|
|
for _, value := range []string{text, reference, "chunk material", "incoming-value"} {
|
|
if !strings.Contains(content, value) {
|
|
t.Fatalf("missing self-contained input %q", value)
|
|
}
|
|
}
|
|
if strings.Contains(content, "configured-option-must-use-native-inspection") {
|
|
t.Fatal("runtime configuration bypassed secret-aware inspection")
|
|
}
|
|
}
|
|
|
|
func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) {
|
|
_, store := filesystemProducerCache(t)
|
|
calls := 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
|
|
prepared := prepareRunnerProfile(t, catalog, profile, engine)
|
|
for _, reuse := range []bool{false, true} {
|
|
evidence := &nativeEvidence{}
|
|
var group promptkit.ExecutionGroup
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1})
|
|
if err != nil || output.FallbackCount != 0 || calls != 1 {
|
|
t.Fatalf("run: %#v %v, calls=%d", output, err, calls)
|
|
}
|
|
if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 {
|
|
t.Fatal("missing chunk/extract/merge/normalization boundaries")
|
|
}
|
|
if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] {
|
|
t.Fatal("cache reuse fabricated a native execution")
|
|
}
|
|
for name, content := range evidence.files {
|
|
if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") {
|
|
continue
|
|
}
|
|
var outcome struct {
|
|
Artifact struct{ Content debugBinaryEnvelope }
|
|
}
|
|
if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) {
|
|
t.Fatalf("missing codec artifact for %s: %s %v", name, content, err)
|
|
}
|
|
}
|
|
if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 {
|
|
t.Fatal("reused artifact bytes not retained")
|
|
}
|
|
var snapshot promptkit.ExecutionGroupSummary
|
|
if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete {
|
|
t.Fatalf("incorrect current-run totals: %#v", snapshot)
|
|
}
|
|
if len(evidence.files["composition.json"]) == 0 {
|
|
t.Fatal("resolved composition unavailable")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) {
|
|
for _, fail := range []bool{false, true} {
|
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
|
probe := newRunnerProbe()
|
|
primary := errors.New("deterministic operation failed")
|
|
producer := runnerNotesProducer{name: "notes", probe: probe}
|
|
if fail {
|
|
producer.failure = primary
|
|
}
|
|
lane := registerRunnerNotes(t, catalog, producer)
|
|
permissive := contracts.ModePermissive
|
|
lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe)
|
|
profile := typedResolutionProfile()
|
|
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
|
|
evidence := &nativeEvidence{}
|
|
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1})
|
|
if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) {
|
|
t.Fatalf("result=%#v %v", output, err)
|
|
}
|
|
found := false
|
|
for name, content := range evidence.files {
|
|
if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") {
|
|
continue
|
|
}
|
|
found = true
|
|
var result struct {
|
|
Outcome contracts.ProducerOutcome
|
|
Error string
|
|
Artifact any
|
|
}
|
|
if err := json.Unmarshal(content, &result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if fail {
|
|
if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil {
|
|
t.Fatalf("fabricated failed artifact: %s", content)
|
|
}
|
|
} else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil {
|
|
t.Fatalf("fallback lost qualification: %s", content)
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("failed/fallback operation not captured")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) {
|
|
primary := errors.New("interpretation evidence failed")
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
fallbacks := 0
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) {
|
|
return nil, nil, &contracts.CaptureError{Cause: primary}
|
|
}, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
|
|
if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" {
|
|
t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks)
|
|
}
|
|
}
|
|
|
|
func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) {
|
|
binding := Binding("configured-module")
|
|
binding.Options = map[string]any{"token": "configuration-secret"}
|
|
validator := Binding("configured-validator")
|
|
validator.Options = map[string]any{"token": "validator-configuration-secret"}
|
|
binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}}
|
|
resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}}
|
|
content, err := json.Marshal(captureComposition(resolved))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") {
|
|
t.Fatalf("composition lost identities or bypassed native inspection: %s", content)
|
|
}
|
|
if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" {
|
|
t.Fatal("capture mutated configured module options")
|
|
}
|
|
}
|