package pipeline import ( "context" "encoding/base64" "encoding/json" "errors" "fmt" "path" "strings" "sync" "sync/atomic" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "go.maximumdirect.net/promptkit" ) type nativeEvidence struct { mu sync.Mutex files map[string][]byte fail func(string) error } func (evidence *nativeEvidence) WriteJSON(name string, value any) error { content, err := json.MarshalIndent(value, "", " ") if err != nil { return err } return evidence.WriteBytes(name, content) } func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error { evidence.mu.Lock() defer evidence.mu.Unlock() if evidence.fail != nil { if err := evidence.fail(name); err != nil { return err } } if evidence.files == nil { evidence.files = make(map[string][]byte) } evidence.files[name] = append([]byte(nil), content...) return nil } func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int { count := 0 for name := range evidence.files { if strings.HasSuffix(name, suffix) { count++ } } return count } func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) { evidence := &nativeEvidence{} var group promptkit.ExecutionGroup started, prepared, providerCalls := 0, 0, 0 initialExecutions := 0 source := strings.Repeat("sk-ordinary-application-text ", 1000) engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { providerCalls++ value := 1 if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") { initialExecutions++ if initialExecutions == 1 { value = 0 } } return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil })) options := contracts.ExecutionOptions{ Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionStarted { started++ } }), OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ }, Capture: evidence, } operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { request, _ := producerCorrection(ctx, []string{"nested assessment"}) request = producerRequest(request, options, true) request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value))) result, err := engine.Run(ctx, request) if captureErr := capture.Finish(result, err, false, false); captureErr != nil { return contracts.CandidateAssessment{}, captureErr } if err != nil { return contracts.CandidateAssessment{}, err } if value == 1 { return acceptedProducerAssessment(ctx, value) } return rejectedProducerAssessment(ctx, value) }, } initial := nativeProducerInitial() initial.APIKey = "request-credential-must-be-omitted" initial.Inputs["source"] = promptkit.Inline(source) initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) { details.Messages[0].Content = "changed by existing preparation hook" } value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil) if err != nil || value != 1 { t.Fatalf("result=%d, %v", value, err) } summary := group.Snapshot() if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete { t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary) } if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete { t.Fatalf("fabricated known usage: %#v", summary.Usage) } if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 { t.Fatalf("missing native results/exchanges: %v", evidence.files) } var initialID string var continuation promptkit.RunResult for name, content := range evidence.files { if strings.Contains(string(content), initial.APIKey) { t.Fatalf("native credential leaked in %s", name) } if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") { t.Fatalf("preparation hook mutated retained evidence: %s", name) } if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) { t.Fatal("selected source content was truncated or rewritten") } if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") { var result promptkit.RunResult if err := json.Unmarshal(content, &result); err != nil { t.Fatal(err) } initialID = result.RunID } if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") { if err := json.Unmarshal(content, &continuation); err != nil { t.Fatal(err) } } } if initialID == "" || continuation.ParentRunID != initialID { t.Fatalf("native parent identity lost: %q %#v", initialID, continuation) } if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") { t.Fatalf("visible rejected exchange history lost: %q", got) } } func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) { _, store := filesystemProducerCache(t) engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) operation := numberCacheOperation(store) if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil { t.Fatal(err) } evidence := &nativeEvidence{} var group promptkit.ExecutionGroup operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe, OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") }, } operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) { t.Fatal("cache hit reassessed") return contracts.CandidateAssessment{}, nil } value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) if err != nil || value != 1 { t.Fatalf("hit=%d, %v", value, err) } if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 { t.Fatal("cache hit fabricated execution/history") } var details promptkit.PreparedRun if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" { t.Fatalf("hit prepared details unavailable: %#v %v", details, err) } if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) { t.Fatal("cache hit not labeled") } } func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) { for _, missing := range []bool{false, true} { evidence := &nativeEvidence{} primary := errors.New("provider stopped") engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return nil, primary })) request := nativeProducerInitial() if missing { request.PromptID = "not-present" } request, capture := CaptureNativeRequest(request, evidence) result, err := engine.Run(context.Background(), request) if err == nil || result != nil { t.Fatalf("expected result-less native failure: %#v, %v", result, err) } if captureErr := capture.Finish(result, err, false, false); captureErr != nil { t.Fatal(captureErr) } if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 { t.Fatal("fabricated result/provider body") } if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] { t.Fatal("prepared availability wrong") } found := false for name, content := range evidence.files { if strings.HasSuffix(name, "/availability.json") { found = true if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) { t.Fatalf("failure evidence unavailable: %s", content) } } } if !found { t.Fatal("missing failure evidence") } } } func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) { evidence := &nativeEvidence{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() finished, assessed, fallbackCalls := 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1}, Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) { if event.Kind == promptkit.ExecutionFinished { finished++ if finished == 2 { cancel() } } }}, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessed++ return rejectedProducerAssessment(ctx, value) }, } _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil }) if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" { t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls) } if nativeEvidenceCount(evidence, "/result.json") != 2 { t.Fatal("undelivered native response lost") } found := false for name, content := range evidence.files { if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) { found = true } } if !found { t.Fatal("undelivered response not labeled unassessed") } } func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) { for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} { t.Run(boundary, func(t *testing.T) { captureErr := errors.New("evidence write failed") evidence := &nativeEvidence{fail: func(name string) error { if strings.Contains(name, boundary) { return captureErr } return nil }} calls, assessed, fallbackCalls := 0, 0, 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence}, Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { assessed++ return rejectedProducerAssessment(ctx, value) }, } _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil }) var recording *contracts.CaptureError if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" { t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls) } if boundary == "request.json" && calls != 0 { t.Fatal("model ran after pre-execution capture failure") } }) } } func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) { evidence := &nativeEvidence{} var group promptkit.ExecutionGroup var prepared atomic.Int64 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil })) const operations = 12 failures := make(chan error, operations) var workers sync.WaitGroup for index := range operations { workers.Go(func() { operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)), OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) }, }, } _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) failures <- err }) } workers.Wait() close(failures) for err := range failures { if err != nil { t.Fatal(err) } } summary := group.Snapshot() if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations { t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json")) } } func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) { content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000) metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"} envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil) decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64) if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] { t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err) } doc := typedTestDocument() doc.Units[0].Text, doc.Metadata = content, metadata if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 { t.Fatal("parsed source content or ordinary metadata was rewritten") } schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)} serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}}) if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) { t.Fatal("available artifact schema was omitted") } } func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) { evidence := &nativeEvidence{} calls := 0 engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { calls++ return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}} _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) { return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil }, nil) if err == nil || calls != 1 { t.Fatalf("missing correction preparation: %v calls=%d", err, calls) } found := false for name, content := range evidence.files { if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") { found = true } } if !found { t.Fatal("constructed correction input lost before preparation") } } func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) { primary, recording := errors.New("provider failed"), errors.New("capture failed") evidence := &nativeEvidence{fail: func(name string) error { if strings.HasSuffix(name, "/prepared.json") { return recording } return nil }} engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { return nil, primary })) operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}} _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil }) if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" { t.Fatalf("failure causes lost: %#v %v", outcome, err) } }