Files
notarius/internal/framework/pipeline/native_capture_test.go

397 lines
17 KiB
Go

package pipeline
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"sync"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type nativeEvidence struct {
mu sync.Mutex
files map[string][]byte
fail func(string) error
}
func (evidence *nativeEvidence) WriteJSON(name string, value any) error {
content, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return evidence.WriteBytes(name, content)
}
func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error {
evidence.mu.Lock()
defer evidence.mu.Unlock()
if evidence.fail != nil {
if err := evidence.fail(name); err != nil {
return err
}
}
if evidence.files == nil {
evidence.files = make(map[string][]byte)
}
evidence.files[name] = append([]byte(nil), content...)
return nil
}
func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int {
count := 0
for name := range evidence.files {
if strings.HasSuffix(name, suffix) {
count++
}
}
return count
}
func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
started, prepared, providerCalls := 0, 0, 0
initialExecutions := 0
source := strings.Repeat("sk-ordinary-application-text ", 1000)
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
providerCalls++
value := 1
if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") {
initialExecutions++
if initialExecutions == 1 {
value = 0
}
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
options := contracts.ExecutionOptions{
Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionStarted {
started++
}
}),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ },
Capture: evidence,
}
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
request, _ := producerCorrection(ctx, []string{"nested assessment"})
request = producerRequest(request, options, true)
request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value)))
result, err := engine.Run(ctx, request)
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
return contracts.CandidateAssessment{}, captureErr
}
if err != nil {
return contracts.CandidateAssessment{}, err
}
if value == 1 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
}
initial := nativeProducerInitial()
initial.APIKey = "request-credential-must-be-omitted"
initial.Inputs["source"] = promptkit.Inline(source)
initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
details.Messages[0].Content = "changed by existing preparation hook"
}
value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil)
if err != nil || value != 1 {
t.Fatalf("result=%d, %v", value, err)
}
summary := group.Snapshot()
if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete {
t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary)
}
if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete {
t.Fatalf("fabricated known usage: %#v", summary.Usage)
}
if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 {
t.Fatalf("missing native results/exchanges: %v", evidence.files)
}
var initialID string
var continuation promptkit.RunResult
for name, content := range evidence.files {
if strings.Contains(string(content), initial.APIKey) {
t.Fatalf("native credential leaked in %s", name)
}
if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") {
t.Fatalf("preparation hook mutated retained evidence: %s", name)
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) {
t.Fatal("selected source content was truncated or rewritten")
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") {
var result promptkit.RunResult
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
initialID = result.RunID
}
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") {
if err := json.Unmarshal(content, &continuation); err != nil {
t.Fatal(err)
}
}
}
if initialID == "" || continuation.ParentRunID != initialID {
t.Fatalf("native parent identity lost: %q %#v", initialID, continuation)
}
if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") {
t.Fatalf("visible rejected exchange history lost: %q", got)
}
}
func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) {
_, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil {
t.Fatal(err)
}
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe,
OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") },
}
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
t.Fatal("cache hit reassessed")
return contracts.CandidateAssessment{}, nil
}
value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 {
t.Fatalf("hit=%d, %v", value, err)
}
if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 {
t.Fatal("cache hit fabricated execution/history")
}
var details promptkit.PreparedRun
if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" {
t.Fatalf("hit prepared details unavailable: %#v %v", details, err)
}
if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) {
t.Fatal("cache hit not labeled")
}
}
func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) {
for _, missing := range []bool{false, true} {
evidence := &nativeEvidence{}
primary := errors.New("provider stopped")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
request := nativeProducerInitial()
if missing {
request.PromptID = "not-present"
}
request, capture := CaptureNativeRequest(request, evidence)
result, err := engine.Run(context.Background(), request)
if err == nil || result != nil {
t.Fatalf("expected result-less native failure: %#v, %v", result, err)
}
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
t.Fatal(captureErr)
}
if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 {
t.Fatal("fabricated result/provider body")
}
if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] {
t.Fatal("prepared availability wrong")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") {
found = true
if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) {
t.Fatalf("failure evidence unavailable: %s", content)
}
}
}
if !found {
t.Fatal("missing failure evidence")
}
}
}
func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) {
evidence := &nativeEvidence{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
finished, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1},
Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
finished++
if finished == 2 {
cancel()
}
}
}},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls)
}
if nativeEvidenceCount(evidence, "/result.json") != 2 {
t.Fatal("undelivered native response lost")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) {
found = true
}
}
if !found {
t.Fatal("undelivered response not labeled unassessed")
}
}
func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) {
for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} {
t.Run(boundary, func(t *testing.T) {
captureErr := errors.New("evidence write failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.Contains(name, boundary) {
return captureErr
}
return nil
}}
calls, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
var recording *contracts.CaptureError
if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls)
}
if boundary == "request.json" && calls != 0 {
t.Fatal("model ran after pre-execution capture failure")
}
})
}
}
func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
var prepared atomic.Int64
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
const operations = 12
failures := make(chan error, operations)
var workers sync.WaitGroup
for index := range operations {
workers.Go(func() {
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment,
Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) },
},
}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
failures <- err
})
}
workers.Wait()
close(failures)
for err := range failures {
if err != nil {
t.Fatal(err)
}
}
summary := group.Snapshot()
if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations {
t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json"))
}
}
func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) {
content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000)
metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"}
envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil)
decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64)
if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] {
t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err)
}
doc := typedTestDocument()
doc.Units[0].Text, doc.Metadata = content, metadata
if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 {
t.Fatal("parsed source content or ordinary metadata was rewritten")
}
schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}})
if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) {
t.Fatal("available artifact schema was omitted")
}
}
func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) {
evidence := &nativeEvidence{}
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil
}, nil)
if err == nil || calls != 1 {
t.Fatalf("missing correction preparation: %v calls=%d", err, calls)
}
found := false
for name, content := range evidence.files {
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") {
found = true
}
}
if !found {
t.Fatal("constructed correction input lost before preparation")
}
}
func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) {
primary, recording := errors.New("provider failed"), errors.New("capture failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.HasSuffix(name, "/prepared.json") {
return recording
}
return nil
}}
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil })
if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" {
t.Fatalf("failure causes lost: %#v %v", outcome, err)
}
}