397 lines
17 KiB
Go
397 lines
17 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
type nativeEvidence struct {
|
|
mu sync.Mutex
|
|
files map[string][]byte
|
|
fail func(string) error
|
|
}
|
|
|
|
func (evidence *nativeEvidence) WriteJSON(name string, value any) error {
|
|
content, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return evidence.WriteBytes(name, content)
|
|
}
|
|
func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error {
|
|
evidence.mu.Lock()
|
|
defer evidence.mu.Unlock()
|
|
if evidence.fail != nil {
|
|
if err := evidence.fail(name); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if evidence.files == nil {
|
|
evidence.files = make(map[string][]byte)
|
|
}
|
|
evidence.files[name] = append([]byte(nil), content...)
|
|
return nil
|
|
}
|
|
|
|
func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int {
|
|
count := 0
|
|
for name := range evidence.files {
|
|
if strings.HasSuffix(name, suffix) {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) {
|
|
evidence := &nativeEvidence{}
|
|
var group promptkit.ExecutionGroup
|
|
started, prepared, providerCalls := 0, 0, 0
|
|
initialExecutions := 0
|
|
source := strings.Repeat("sk-ordinary-application-text ", 1000)
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
providerCalls++
|
|
value := 1
|
|
if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") {
|
|
initialExecutions++
|
|
if initialExecutions == 1 {
|
|
value = 0
|
|
}
|
|
}
|
|
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
|
|
}))
|
|
options := contracts.ExecutionOptions{
|
|
Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionStarted {
|
|
started++
|
|
}
|
|
}),
|
|
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ },
|
|
Capture: evidence,
|
|
}
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options,
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
request, _ := producerCorrection(ctx, []string{"nested assessment"})
|
|
request = producerRequest(request, options, true)
|
|
request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value)))
|
|
result, err := engine.Run(ctx, request)
|
|
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
|
|
return contracts.CandidateAssessment{}, captureErr
|
|
}
|
|
if err != nil {
|
|
return contracts.CandidateAssessment{}, err
|
|
}
|
|
if value == 1 {
|
|
return acceptedProducerAssessment(ctx, value)
|
|
}
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
initial := nativeProducerInitial()
|
|
initial.APIKey = "request-credential-must-be-omitted"
|
|
initial.Inputs["source"] = promptkit.Inline(source)
|
|
initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
|
|
details.Messages[0].Content = "changed by existing preparation hook"
|
|
}
|
|
value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil)
|
|
if err != nil || value != 1 {
|
|
t.Fatalf("result=%d, %v", value, err)
|
|
}
|
|
summary := group.Snapshot()
|
|
if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete {
|
|
t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary)
|
|
}
|
|
if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete {
|
|
t.Fatalf("fabricated known usage: %#v", summary.Usage)
|
|
}
|
|
if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 {
|
|
t.Fatalf("missing native results/exchanges: %v", evidence.files)
|
|
}
|
|
var initialID string
|
|
var continuation promptkit.RunResult
|
|
for name, content := range evidence.files {
|
|
if strings.Contains(string(content), initial.APIKey) {
|
|
t.Fatalf("native credential leaked in %s", name)
|
|
}
|
|
if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") {
|
|
t.Fatalf("preparation hook mutated retained evidence: %s", name)
|
|
}
|
|
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) {
|
|
t.Fatal("selected source content was truncated or rewritten")
|
|
}
|
|
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") {
|
|
var result promptkit.RunResult
|
|
if err := json.Unmarshal(content, &result); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
initialID = result.RunID
|
|
}
|
|
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") {
|
|
if err := json.Unmarshal(content, &continuation); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
if initialID == "" || continuation.ParentRunID != initialID {
|
|
t.Fatalf("native parent identity lost: %q %#v", initialID, continuation)
|
|
}
|
|
if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") {
|
|
t.Fatalf("visible rejected exchange history lost: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) {
|
|
_, store := filesystemProducerCache(t)
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
operation := numberCacheOperation(store)
|
|
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
evidence := &nativeEvidence{}
|
|
var group promptkit.ExecutionGroup
|
|
operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe,
|
|
OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") },
|
|
}
|
|
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
|
|
t.Fatal("cache hit reassessed")
|
|
return contracts.CandidateAssessment{}, nil
|
|
}
|
|
value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
|
if err != nil || value != 1 {
|
|
t.Fatalf("hit=%d, %v", value, err)
|
|
}
|
|
if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 {
|
|
t.Fatal("cache hit fabricated execution/history")
|
|
}
|
|
var details promptkit.PreparedRun
|
|
if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" {
|
|
t.Fatalf("hit prepared details unavailable: %#v %v", details, err)
|
|
}
|
|
if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) {
|
|
t.Fatal("cache hit not labeled")
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) {
|
|
for _, missing := range []bool{false, true} {
|
|
evidence := &nativeEvidence{}
|
|
primary := errors.New("provider stopped")
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return nil, primary
|
|
}))
|
|
request := nativeProducerInitial()
|
|
if missing {
|
|
request.PromptID = "not-present"
|
|
}
|
|
request, capture := CaptureNativeRequest(request, evidence)
|
|
result, err := engine.Run(context.Background(), request)
|
|
if err == nil || result != nil {
|
|
t.Fatalf("expected result-less native failure: %#v, %v", result, err)
|
|
}
|
|
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
|
|
t.Fatal(captureErr)
|
|
}
|
|
if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 {
|
|
t.Fatal("fabricated result/provider body")
|
|
}
|
|
if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] {
|
|
t.Fatal("prepared availability wrong")
|
|
}
|
|
found := false
|
|
for name, content := range evidence.files {
|
|
if strings.HasSuffix(name, "/availability.json") {
|
|
found = true
|
|
if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) {
|
|
t.Fatalf("failure evidence unavailable: %s", content)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("missing failure evidence")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) {
|
|
evidence := &nativeEvidence{}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
finished, assessed, fallbackCalls := 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1},
|
|
Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) {
|
|
if event.Kind == promptkit.ExecutionFinished {
|
|
finished++
|
|
if finished == 2 {
|
|
cancel()
|
|
}
|
|
}
|
|
}},
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
|
|
if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" {
|
|
t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls)
|
|
}
|
|
if nativeEvidenceCount(evidence, "/result.json") != 2 {
|
|
t.Fatal("undelivered native response lost")
|
|
}
|
|
found := false
|
|
for name, content := range evidence.files {
|
|
if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("undelivered response not labeled unassessed")
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) {
|
|
for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} {
|
|
t.Run(boundary, func(t *testing.T) {
|
|
captureErr := errors.New("evidence write failed")
|
|
evidence := &nativeEvidence{fail: func(name string) error {
|
|
if strings.Contains(name, boundary) {
|
|
return captureErr
|
|
}
|
|
return nil
|
|
}}
|
|
calls, assessed, fallbackCalls := 0, 0, 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence},
|
|
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
|
|
assessed++
|
|
return rejectedProducerAssessment(ctx, value)
|
|
},
|
|
}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
|
|
var recording *contracts.CaptureError
|
|
if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" {
|
|
t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls)
|
|
}
|
|
if boundary == "request.json" && calls != 0 {
|
|
t.Fatal("model ran after pre-execution capture failure")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) {
|
|
evidence := &nativeEvidence{}
|
|
var group promptkit.ExecutionGroup
|
|
var prepared atomic.Int64
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
|
|
}))
|
|
const operations = 12
|
|
failures := make(chan error, operations)
|
|
var workers sync.WaitGroup
|
|
for index := range operations {
|
|
workers.Go(func() {
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment,
|
|
Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)),
|
|
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) },
|
|
},
|
|
}
|
|
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
|
|
failures <- err
|
|
})
|
|
}
|
|
workers.Wait()
|
|
close(failures)
|
|
for err := range failures {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
summary := group.Snapshot()
|
|
if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations {
|
|
t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json"))
|
|
}
|
|
}
|
|
|
|
func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) {
|
|
content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000)
|
|
metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"}
|
|
envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil)
|
|
decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64)
|
|
if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] {
|
|
t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err)
|
|
}
|
|
doc := typedTestDocument()
|
|
doc.Units[0].Text, doc.Metadata = content, metadata
|
|
if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 {
|
|
t.Fatal("parsed source content or ordinary metadata was rewritten")
|
|
}
|
|
schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
|
|
serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}})
|
|
if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) {
|
|
t.Fatal("available artifact schema was omitted")
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) {
|
|
evidence := &nativeEvidence{}
|
|
calls := 0
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
calls++
|
|
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
|
|
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil
|
|
}, nil)
|
|
if err == nil || calls != 1 {
|
|
t.Fatalf("missing correction preparation: %v calls=%d", err, calls)
|
|
}
|
|
found := false
|
|
for name, content := range evidence.files {
|
|
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("constructed correction input lost before preparation")
|
|
}
|
|
}
|
|
|
|
func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) {
|
|
primary, recording := errors.New("provider failed"), errors.New("capture failed")
|
|
evidence := &nativeEvidence{fail: func(name string) error {
|
|
if strings.HasSuffix(name, "/prepared.json") {
|
|
return recording
|
|
}
|
|
return nil
|
|
}}
|
|
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
|
return nil, primary
|
|
}))
|
|
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
|
|
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil })
|
|
if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" {
|
|
t.Fatalf("failure causes lost: %#v %v", outcome, err)
|
|
}
|
|
}
|