Capture native execution facts and complete scoped debug evidence

This commit is contained in:
2026-11-22 03:19:43 +00:00
parent 8680060400
commit 9f66421f4f
24 changed files with 1688 additions and 543 deletions

View File

@@ -6,7 +6,10 @@ import (
"errors"
"fmt"
"reflect"
"strconv"
"sync/atomic"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -30,6 +33,10 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i
}
validator.position = index + 1
result, err := invoke(ctx, validator)
var captureErr *contracts.CaptureError
if errors.As(err, &captureErr) {
return assessment, errors.Join(ctx.Err(), err)
}
if canceled := ctx.Err(); canceled != nil {
return assessment, canceled
}
@@ -70,6 +77,7 @@ func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
// typedCandidateAssessor binds one operation's complete chain. Each invocation
// captures the new candidate once and decodes a detached view for each validator.
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
var candidateNumber atomic.Uint64
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
@@ -81,12 +89,21 @@ func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidat
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
if capture != nil {
if err := capture.WriteBytes("candidate.json", content); err != nil {
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
}
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
request.execution.Capture = validatorCapture
switch validator.resolved.Target {
case ValidatorTargetTyped:
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
@@ -153,6 +170,7 @@ func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument,
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
// The runner's plan callback owns materialization before this boundary.
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
var candidateNumber atomic.Uint64
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
@@ -167,12 +185,21 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
target := target
target.chunks = chunks
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
if capture != nil {
if err := capture.WriteBytes("candidate.json", content); err != nil {
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
}
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
request.execution.Capture = validatorCapture
switch validator.resolved.Target {
case ValidatorTargetChunk:
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
@@ -193,3 +220,17 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
})
}
}
func captureValidationFinding(writer contracts.EvidenceWriter, finding contracts.ValidationResult, failure error) error {
if writer == nil {
return failure
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
if err := writer.WriteJSON("finding.json", map[string]any{"result": finding, "correction_guidance": finding.CorrectionGuidance, "error": errorText}); err != nil {
return errors.Join(failure, &contracts.CaptureError{Cause: err})
}
return failure
}

View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -124,6 +125,56 @@ func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
return entry
}
func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) {
evidence := &nativeEvidence{}
content := strings.Repeat("sk-ordinary content ", 1000)
candidate := codecNotes{Items: []string{content}}
validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
if request.Execution.Capture == nil {
t.Fatal("validator has no scoped capture")
}
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil
})
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}})
for range 2 {
assessment, err := assess(context.Background(), candidate)
if err != nil || assessment.Accepted() {
t.Fatalf("assessment=%#v %v", assessment, err)
}
}
if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 {
t.Fatal("revised candidates or findings were overwritten")
}
for name, data := range evidence.files {
if strings.HasSuffix(name, "/candidate.json") {
decoded, err := assessmentNotesCodec(t).decodeCandidate(data)
if err != nil || !reflect.DeepEqual(decoded, candidate) {
t.Fatalf("codec candidate changed: %#v, %v", decoded, err)
}
}
if strings.HasSuffix(name, "/finding.json") {
var finding struct {
Guidance string `json:"correction_guidance"`
}
if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content {
t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err)
}
}
}
}
func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) {
primary := errors.New("capture failed")
called := 0
_, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
called++
return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary}
})
if called != 1 || !errors.Is(err, primary) {
t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err)
}
}
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
doc := typedTestDocument()

View File

@@ -1,22 +1,13 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"strings"
"sync"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -36,21 +27,19 @@ func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
type debugTimedEnvelope struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Attempt int `json:"attempt,omitempty"`
AttemptKind string `json:"attempt_kind,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"`
Error string `json:"error,omitempty"`
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
Error string `json:"error,omitempty"`
}
type debugBinaryEnvelope struct {
Text string `json:"text,omitempty"`
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
@@ -61,7 +50,6 @@ type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw debugBinaryEnvelope `json:"raw,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
@@ -109,244 +97,6 @@ type debugSerializedOutput struct {
Content debugBinaryEnvelope `json:"content"`
}
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMPromptArtifact struct {
CallID string `json:"call_id"`
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
}
type debugLLMResponseArtifact struct {
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugValidationCall struct {
ValidatorName string `json:"validator_name"`
Request any `json:"request"`
Result contracts.ValidationResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMClient struct {
inner contracts.StructuredLLMClient
recorder DebugRecorder
mu sync.Mutex
counter int
}
type debugLLMScope struct {
prefix string
parent *debugLLMScope
mu sync.Mutex
calls []debugLLMCallReference
}
type debugLLMScopeContextKey struct{}
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
if client == nil || recorder == nil || !recorder.Enabled() {
return client
}
if _, ok := client.(*debugLLMClient); ok {
return client
}
return &debugLLMClient{inner: client, recorder: synchronizedDebugRecorder(recorder)}
}
// WithDebugLLMRecording decorates a shared LLM client so calls made by
// construction-injected modules participate in the run's debug recording.
func WithDebugLLMRecording(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
return wrapDebugLLMClient(client, recorder)
}
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.mu.Lock()
client.counter++
callID := fmt.Sprintf("%04d", client.counter)
client.mu.Unlock()
started := time.Now().UTC()
response, err := client.inner.CompleteStructured(ctx, req, out)
completed := time.Now().UTC()
errorText := ""
if err != nil {
errorText = err.Error()
}
scopePrefix := cleanDebugPath(req.StageName)
if req.StageName == "" {
scopePrefix = "llm"
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scopePrefix = scope.prefix
}
promptPath := ""
var writeErr error
if response.Debug != nil && response.Debug.Prompt != nil {
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMPromptArtifact{
CallID: callID,
Prompt: response.Debug.Prompt,
},
}))
}
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
responseMaterial := debugResponseMaterial(response)
fallbackMaterial := debugCompletionFallback(response)
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
writeErr = errors.Join(writeErr, contentErr)
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMResponseArtifact{
CallID: callID,
Response: responseForArtifact,
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
},
Error: errorText,
}))
callRef := debugLLMCallReference{
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
ResponseContentPath: responseContentPath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
}
if err != nil {
return response, err
}
if writeErr != nil {
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
}
return response, err
}
func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
scope := &debugLLMScope{
prefix: prefix,
parent: debugLLMScopeFromContext(ctx),
}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func withIsolatedDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
scope := &debugLLMScope{prefix: prefix}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope {
if ctx == nil {
return nil
}
scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope)
return scope
}
func (scope *debugLLMScope) record(ref debugLLMCallReference) {
if scope == nil {
return
}
scope.mu.Lock()
scope.calls = append(scope.calls, ref)
scope.mu.Unlock()
if scope.parent != nil {
scope.parent.record(ref)
}
}
func (scope *debugLLMScope) references() []debugLLMCallReference {
if scope == nil {
return nil
}
scope.mu.Lock()
defer scope.mu.Unlock()
if len(scope.calls) == 0 {
return nil
}
out := make([]debugLLMCallReference, len(scope.calls))
copy(out, scope.calls)
return out
}
func cleanDebugPath(value string) string {
parts := strings.Split(value, "/")
out := make([]string, 0, len(parts))
for _, part := range parts {
out = append(out, fileio.EncodePathComponent(part))
}
return strings.Join(out, "/")
}
func debugFirstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
if recorder == nil || !recorder.Enabled() {
return nil
@@ -363,60 +113,17 @@ func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnv
return recorder.WriteJSON(name, envelope)
}
func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope {
if scope != nil {
envelope.LLMCalls = scope.references()
}
return envelope
}
func writeDebugAttempt(recorder DebugRecorder, attemptPath string, envelope debugTimedEnvelope, scope *debugLLMScope) error {
return writeDebugTimed(recorder, attemptPath+".json", debugEnvelopeWithLLMCalls(envelope, scope))
}
type attemptTerminalRecorder struct {
recorder DebugRecorder
path string
label string
scope *debugLLMScope
envelope debugTimedEnvelope
}
type attemptDebugPersistenceError struct {
label string
err error
}
func (e *attemptDebugPersistenceError) Error() string {
return fmt.Sprintf("write %s attempt debug artifact: %v", e.label, e.err)
}
func (e *attemptDebugPersistenceError) Unwrap() error { return e.err }
func newAttemptTerminalRecorder(recorder DebugRecorder, attemptPath, label string, scope *debugLLMScope, envelope debugTimedEnvelope) attemptTerminalRecorder {
return attemptTerminalRecorder{recorder: recorder, path: attemptPath, label: label, scope: scope, envelope: envelope}
}
func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
envelope := r.envelope
envelope.Payload = payload
if terminalErr != nil {
envelope.Error = terminalErr.Error()
}
if err := writeDebugAttempt(r.recorder, r.path, envelope, r.scope); err != nil {
debugErr := &attemptDebugPersistenceError{label: r.label, err: err}
return errors.Join(terminalErr, debugErr)
}
return terminalErr
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
content = redactSecretBytes(content)
text := ""
if utf8.Valid(content) {
text = string(content)
}
return debugBinaryEnvelope{
Text: text,
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: debugContentDigest(content),
MediaType: mediaType,
Metadata: redactSensitiveMap(metadata),
Metadata: metadata,
}
}
@@ -430,7 +137,7 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnitsForDebug(doc.Units),
Metadata: redactSensitiveMap(doc.Metadata),
Metadata: doc.Metadata,
}
}
@@ -442,7 +149,7 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnitsForDebug(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
Metadata: chunk.Metadata,
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
}
@@ -457,9 +164,9 @@ func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
cloned[i] = source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: string(redactSecretBytes([]byte(unit.Text))),
Text: unit.Text,
Ref: unit.Ref,
Metadata: redactSensitiveMap(unit.Metadata),
Metadata: unit.Metadata,
}
}
return cloned
@@ -510,7 +217,6 @@ func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema)
schema.JSONSchema = nil
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(output.Artifact.Content)
return debugSerializedOutput{
@@ -552,189 +258,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
return out
}
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
return req.DebugSummary()
}
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
return debugStructuredCompletionResponse{
Content: string(response.Content),
Provider: response.Provider,
Model: response.Model,
ProfileID: response.ProfileID,
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
}
}
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
if response.Debug != nil && response.Debug.Response != nil {
return nil
}
fallback := debugCompletionResponse(response)
if fallback.Content == "" &&
fallback.Provider == "" &&
fallback.Model == "" &&
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil
}
return &fallback
}
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
responseCopy := cloneDebugResponseWithoutContent(response)
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
content := ""
if response != nil {
content = response.Content
}
if content == "" && fallback != nil {
content = fallback.Content
}
if content == "" {
return "", responseCopy, fallbackCopy, nil
}
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
if recorder == nil || !recorder.Enabled() {
return contentPath, responseCopy, fallbackCopy, nil
}
if err := recorder.WriteBytes(contentPath, data); err != nil {
return contentPath, responseCopy, fallbackCopy, err
}
return contentPath, responseCopy, fallbackCopy, nil
}
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
if response == nil {
return nil
}
clone := *response
clone.Content = ""
return &clone
}
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
if fallback == nil {
return nil
}
clone := *fallback
clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone
}
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
raw := []byte(content)
if json.Valid(raw) {
var formatted bytes.Buffer
if err := json.Indent(&formatted, raw, "", " "); err == nil {
formatted.WriteByte('\n')
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
}
}
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
}
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
if response.Debug == nil {
return nil
}
return response.Debug.Response
}
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
if response.Debug == nil || response.Debug.Response == nil {
return ""
}
return response.Debug.Response.ModelName
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message)))
result.CorrectionGuidance = ""
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
return result
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
var secretPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
}
func redactSecretBytes(content []byte) []byte {
if len(content) == 0 || !utf8.Valid(content) {
return append([]byte(nil), content...)
}
text := string(content)
for _, pattern := range secretPatterns {
text = pattern.ReplaceAllString(text, "[REDACTED]")
}
return []byte(text)
}
func redactSensitiveMap(values map[string]any) map[string]any {
if len(values) == 0 {
return nil
}
out := make(map[string]any, len(values))
for key, value := range values {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
continue
}
out[key] = redactSensitiveValue(value)
}
return out
}
func redactSensitiveValue(value any) any {
switch typed := value.(type) {
case string:
return string(redactSecretBytes([]byte(typed)))
case map[string]any:
return redactSensitiveMap(typed)
case map[string]string:
out := make(map[string]string, len(typed))
for key, value := range typed {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
} else {
out[key] = string(redactSecretBytes([]byte(value)))
}
}
return out
default:
return value
}
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
return strings.Contains(key, "api_key") ||
strings.Contains(key, "apikey") ||
strings.Contains(key, "authorization") ||
strings.Contains(key, "bearer") ||
strings.Contains(key, "password") ||
strings.Contains(key, "secret") ||
strings.Contains(key, "token")
}

View File

@@ -0,0 +1,255 @@
package pipeline
import (
"context"
"errors"
"fmt"
"maps"
"path"
"strings"
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
// NativeCapture retains only one request's prepared snapshot. Events are written
// incrementally, not accumulated as a second execution/usage ledger. It never
// prepares or executes a request itself.
type NativeCapture struct {
writer contracts.EvidenceWriter
mu sync.Mutex
prepared *promptkit.PreparedRun
runID string
events uint64
err error
}
// CaptureNativeRequest composes existing hooks and enables native exchange
// capture only when evidence was requested. Request credentials remain excluded
// by the native request's own JSON representation.
func CaptureNativeRequest(request promptkit.RunRequest, writer contracts.EvidenceWriter) (promptkit.RunRequest, *NativeCapture) {
if writer == nil {
return request, nil
}
capture := &NativeCapture{writer: writer}
capture.remember(writer.WriteJSON("request.json", request))
request.CaptureExchange = true
existing := request.OnPrepared
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
// No serialization or filesystem I/O while native admission is held.
// Native hands off a detached value. Copy mutable fields only when an
// existing callback also owns that value and may change it.
if existing != nil {
capture.RetainPrepared(copyPreparedEvidence(details))
existing(ctx, details)
} else {
capture.RetainPrepared(details)
}
}
request.Observer = promptkit.ComposeObservers(request.Observer, capture.observe)
return request, capture
}
// RetainPrepared uses the already available split-preparation details on cache
// paths, including hits on which OnPrepared and native events never occur.
func (capture *NativeCapture) RetainPrepared(details promptkit.PreparedRun) {
if capture == nil {
return
}
capture.mu.Lock()
defer capture.mu.Unlock()
capture.prepared = &details
}
func (capture *NativeCapture) remember(err error) {
if capture == nil || err == nil {
return
}
capture.mu.Lock()
defer capture.mu.Unlock()
if capture.err == nil {
capture.err = err
}
}
func (capture *NativeCapture) observe(event promptkit.ExecutionEvent) {
capture.mu.Lock()
defer capture.mu.Unlock()
capture.runID = event.RunID
capture.events++
err := capture.writer.WriteJSON(path.Join("native", event.RunID, "events", fmt.Sprintf("%06d.json", capture.events)), event)
if capture.err == nil {
capture.err = err
}
}
// Err exposes synchronous callback/write failures at the next normal boundary.
func (capture *NativeCapture) Err() error {
if capture == nil {
return nil
}
capture.mu.Lock()
defer capture.mu.Unlock()
if capture.err == nil {
return nil
}
return &contracts.CaptureError{Cause: capture.err}
}
// Finish records available evidence even when native execution returned no
// result. Unassessed marks a response withheld from the application by native
// coordination; cacheHit never fabricates an execution or historical result.
func (capture *NativeCapture) Finish(result *promptkit.RunResult, nativeErr error, unassessed, cacheHit bool) error {
if capture == nil {
return nil
}
capture.mu.Lock()
prepared, runID := capture.prepared, capture.runID
capture.prepared = nil
capture.mu.Unlock()
if result != nil {
runID = result.RunID
}
prefix := "unexecuted"
if runID != "" {
prefix = path.Join("native", runID)
}
writeJSON := func(name string, value any) {
capture.remember(capture.writer.WriteJSON(path.Join(prefix, name), value))
}
writeText := func(name, value string) {
capture.remember(capture.writer.WriteBytes(path.Join(prefix, name), []byte(value)))
}
if prepared != nil {
writeJSON("prepared.json", prepared)
writeText("prepared-messages.txt", readableNativeMessages("Prepared request messages", prepared.Messages))
}
if result != nil {
writeJSON("result.json", result)
writeText("response.txt", result.RawOutput)
if result.Exchange != nil {
details := result.Exchange.Details()
writeJSON("exchange.json", details)
writeText("messages.txt", readableNativeMessages("Completed exchange", details.Messages))
}
}
errorText := ""
if nativeErr != nil {
errorText = nativeErr.Error()
}
writeJSON("availability.json", map[string]any{
"run_id": runID, "prepared_available": prepared != nil,
"result_available": result != nil, "unassessed": unassessed,
"cache_hit": cacheHit, "error": errorText,
})
return capture.Err()
}
func readableNativeMessages(label string, messages []promptkit.RenderedMessage) string {
var text strings.Builder
text.WriteString(label + "\n")
for _, message := range messages {
text.WriteString("\n[" + message.Role + "]\n")
text.WriteString(message.Content)
text.WriteByte('\n')
}
return text.String()
}
type scopedEvidence struct {
writer contracts.EvidenceWriter
prefix string
}
func (scope scopedEvidence) WriteJSON(name string, value any) error {
return scope.writer.WriteJSON(path.Join(scope.prefix, name), value)
}
func (scope scopedEvidence) WriteBytes(name string, value []byte) error {
return scope.writer.WriteBytes(path.Join(scope.prefix, name), value)
}
func scopeEvidence(writer contracts.EvidenceWriter, prefix string) contracts.EvidenceWriter {
if writer == nil {
return nil
}
return scopedEvidence{writer: writer, prefix: prefix}
}
func operationExecution(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk) contracts.ExecutionOptions {
options := input.Execution
writer := options.Capture
if writer == nil && input.Debug != nil && input.Debug.Enabled() {
writer = input.Debug
}
prefix := "operations"
if origin.StepID != "" {
prefix = path.Join(prefix, "steps", fileio.EncodePathComponent(origin.StepID))
}
prefix = path.Join(prefix, string(origin.Stage))
if origin.LaneID != "" {
prefix = path.Join(prefix, "lanes", fileio.EncodePathComponent(origin.LaneID))
}
if chunk != nil {
prefix = path.Join(prefix, "chunks", fileio.EncodePathComponent(chunk.ID))
}
prefix = path.Join(prefix, "modules", fileio.EncodePathComponent(origin.ModuleKey))
options.Capture = scopeEvidence(writer, prefix)
return options
}
// Native schema/extra-parameter documents are JSON values. Copy their mutable
// containers, without serializing large message bodies in OnPrepared.
func copyEvidenceJSON(value any) any {
switch value := value.(type) {
case map[string]any:
copy := make(map[string]any, len(value))
for key, item := range value {
copy[key] = copyEvidenceJSON(item)
}
return copy
case []any:
copy := make([]any, len(value))
for index, item := range value {
copy[index] = copyEvidenceJSON(item)
}
return copy
default:
return value
}
}
func copyPreparedEvidence(details promptkit.PreparedRun) promptkit.PreparedRun {
details.InputHashes = maps.Clone(details.InputHashes)
if details.EffectiveModelParams.ExtraParams != nil {
details.EffectiveModelParams.ExtraParams = copyEvidenceJSON(details.EffectiveModelParams.ExtraParams).(map[string]any)
}
details.OutputContract.Schema = copyEvidenceJSON(details.OutputContract.Schema)
if details.StructuredOutput != nil {
structured := *details.StructuredOutput
if structured.JSONSchema != nil {
schema := *structured.JSONSchema
schema.Schema = copyEvidenceJSON(schema.Schema)
structured.JSONSchema = &schema
}
details.StructuredOutput = &structured
}
messages := make([]promptkit.RenderedMessage, len(details.Messages))
for index, message := range details.Messages {
messages[index] = message
if message.CacheControl != nil {
hint := *message.CacheControl
messages[index].CacheControl = &hint
}
}
details.Messages = messages
return details
}
func captureFailure(primary, recording error) error {
if recording == nil {
return primary
}
return errors.Join(primary, recording)
}

View File

@@ -0,0 +1,396 @@
package pipeline
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"sync"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type nativeEvidence struct {
mu sync.Mutex
files map[string][]byte
fail func(string) error
}
func (evidence *nativeEvidence) WriteJSON(name string, value any) error {
content, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return evidence.WriteBytes(name, content)
}
func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error {
evidence.mu.Lock()
defer evidence.mu.Unlock()
if evidence.fail != nil {
if err := evidence.fail(name); err != nil {
return err
}
}
if evidence.files == nil {
evidence.files = make(map[string][]byte)
}
evidence.files[name] = append([]byte(nil), content...)
return nil
}
func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int {
count := 0
for name := range evidence.files {
if strings.HasSuffix(name, suffix) {
count++
}
}
return count
}
func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
started, prepared, providerCalls := 0, 0, 0
initialExecutions := 0
source := strings.Repeat("sk-ordinary-application-text ", 1000)
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
providerCalls++
value := 1
if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") {
initialExecutions++
if initialExecutions == 1 {
value = 0
}
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
options := contracts.ExecutionOptions{
Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionStarted {
started++
}
}),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ },
Capture: evidence,
}
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
request, _ := producerCorrection(ctx, []string{"nested assessment"})
request = producerRequest(request, options, true)
request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value)))
result, err := engine.Run(ctx, request)
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
return contracts.CandidateAssessment{}, captureErr
}
if err != nil {
return contracts.CandidateAssessment{}, err
}
if value == 1 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
}
initial := nativeProducerInitial()
initial.APIKey = "request-credential-must-be-omitted"
initial.Inputs["source"] = promptkit.Inline(source)
initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
details.Messages[0].Content = "changed by existing preparation hook"
}
value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil)
if err != nil || value != 1 {
t.Fatalf("result=%d, %v", value, err)
}
summary := group.Snapshot()
if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete {
t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary)
}
if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete {
t.Fatalf("fabricated known usage: %#v", summary.Usage)
}
if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 {
t.Fatalf("missing native results/exchanges: %v", evidence.files)
}
var initialID string
var continuation promptkit.RunResult
for name, content := range evidence.files {
if strings.Contains(string(content), initial.APIKey) {
t.Fatalf("native credential leaked in %s", name)
}
if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") {
t.Fatalf("preparation hook mutated retained evidence: %s", name)
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) {
t.Fatal("selected source content was truncated or rewritten")
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") {
var result promptkit.RunResult
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
initialID = result.RunID
}
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") {
if err := json.Unmarshal(content, &continuation); err != nil {
t.Fatal(err)
}
}
}
if initialID == "" || continuation.ParentRunID != initialID {
t.Fatalf("native parent identity lost: %q %#v", initialID, continuation)
}
if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") {
t.Fatalf("visible rejected exchange history lost: %q", got)
}
}
func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) {
_, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil {
t.Fatal(err)
}
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe,
OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") },
}
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
t.Fatal("cache hit reassessed")
return contracts.CandidateAssessment{}, nil
}
value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 {
t.Fatalf("hit=%d, %v", value, err)
}
if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 {
t.Fatal("cache hit fabricated execution/history")
}
var details promptkit.PreparedRun
if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" {
t.Fatalf("hit prepared details unavailable: %#v %v", details, err)
}
if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) {
t.Fatal("cache hit not labeled")
}
}
func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) {
for _, missing := range []bool{false, true} {
evidence := &nativeEvidence{}
primary := errors.New("provider stopped")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
request := nativeProducerInitial()
if missing {
request.PromptID = "not-present"
}
request, capture := CaptureNativeRequest(request, evidence)
result, err := engine.Run(context.Background(), request)
if err == nil || result != nil {
t.Fatalf("expected result-less native failure: %#v, %v", result, err)
}
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
t.Fatal(captureErr)
}
if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 {
t.Fatal("fabricated result/provider body")
}
if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] {
t.Fatal("prepared availability wrong")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") {
found = true
if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) {
t.Fatalf("failure evidence unavailable: %s", content)
}
}
}
if !found {
t.Fatal("missing failure evidence")
}
}
}
func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) {
evidence := &nativeEvidence{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
finished, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1},
Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
finished++
if finished == 2 {
cancel()
}
}
}},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls)
}
if nativeEvidenceCount(evidence, "/result.json") != 2 {
t.Fatal("undelivered native response lost")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) {
found = true
}
}
if !found {
t.Fatal("undelivered response not labeled unassessed")
}
}
func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) {
for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} {
t.Run(boundary, func(t *testing.T) {
captureErr := errors.New("evidence write failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.Contains(name, boundary) {
return captureErr
}
return nil
}}
calls, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
var recording *contracts.CaptureError
if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls)
}
if boundary == "request.json" && calls != 0 {
t.Fatal("model ran after pre-execution capture failure")
}
})
}
}
func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
var prepared atomic.Int64
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
const operations = 12
failures := make(chan error, operations)
var workers sync.WaitGroup
for index := range operations {
workers.Go(func() {
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment,
Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) },
},
}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
failures <- err
})
}
workers.Wait()
close(failures)
for err := range failures {
if err != nil {
t.Fatal(err)
}
}
summary := group.Snapshot()
if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations {
t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json"))
}
}
func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) {
content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000)
metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"}
envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil)
decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64)
if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] {
t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err)
}
doc := typedTestDocument()
doc.Units[0].Text, doc.Metadata = content, metadata
if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 {
t.Fatal("parsed source content or ordinary metadata was rewritten")
}
schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}})
if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) {
t.Fatal("available artifact schema was omitted")
}
}
func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) {
evidence := &nativeEvidence{}
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil
}, nil)
if err == nil || calls != 1 {
t.Fatalf("missing correction preparation: %v calls=%d", err, calls)
}
found := false
for name, content := range evidence.files {
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") {
found = true
}
}
if !found {
t.Fatal("constructed correction input lost before preparation")
}
}
func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) {
primary, recording := errors.New("provider failed"), errors.New("capture failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.HasSuffix(name, "/prepared.json") {
return recording
}
return nil
}}
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil })
if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" {
t.Fatalf("failure causes lost: %#v %v", outcome, err)
}
}

View File

@@ -0,0 +1,115 @@
package pipeline
import (
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func captureOperationInputs(execution contracts.ExecutionOptions, target typedValidationTarget, binding ModuleBinding, incoming []contracts.SerializedArtifact) error {
if execution.Capture == nil {
return nil
}
references := make(map[string]any, len(target.references.Slots))
for name, slot := range target.references.Slots {
items := make([]any, 0, len(slot.Items))
for _, item := range slot.Items {
items = append(items, map[string]any{"reference": item, "content": debugContentEnvelope(item.Content, item.MediaType, nil, nil)})
}
references[name] = map[string]any{"declaration": slot.Slot, "items": items}
}
inputs := make([]debugSerializedOutput, len(incoming))
for index, artifact := range incoming {
inputs[index] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
}
var chunk any
if target.chunk != nil {
chunk = debugSourceChunkEnvelope(*target.chunk)
}
return execution.RecordEvidence("inputs.json", map[string]any{
"source": debugSourceDocumentEnvelope(target.source),
"source_input": debugContentEnvelope(target.sourceInput.Content, target.sourceInput.MediaType, nil, nil),
"chunk": chunk, "chunks": debugSourceChunkEnvelopes(target.chunks), "references": references,
"incoming_artifacts": inputs, "module": binding.Module, "llm_profile": binding.LLMProfile, "metadata": target.metadata,
})
}
// Configured options belong to ConfigKit's secret-aware snapshot, not another
// direct serialization of runtime configuration inside composition evidence.
func captureComposition(resolved ResolvedPipeline) ResolvedPipeline {
resolved = cloneResolvedPipeline(resolved)
resolved.Input, resolved.Chunk, resolved.Output = captureBinding(resolved.Input), captureBinding(resolved.Chunk), captureBinding(resolved.Output)
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
lane.Extract, lane.Merge, lane.Normalize = captureBinding(lane.Extract), captureBinding(lane.Merge), captureBinding(lane.Normalize)
for index := range lane.Validators {
lane.Validators[index] = captureBinding(lane.Validators[index])
}
}
}
for index := range resolved.ValidatorChains {
for validatorIndex := range resolved.ValidatorChains[index].Validators {
resolved.ValidatorChains[index].Validators[validatorIndex].Binding = captureBinding(resolved.ValidatorChains[index].Validators[validatorIndex].Binding)
}
}
return resolved
}
func captureBinding(binding ModuleBinding) ModuleBinding {
binding.Options = nil
for index := range binding.Validators.Validators {
binding.Validators.Validators[index] = captureBinding(binding.Validators.Validators[index])
}
return binding
}
func captureOperationResult(execution contracts.ExecutionOptions, artifact contracts.SerializedArtifact, outcome contracts.ProducerOutcome, failure error) error {
if execution.Capture == nil {
return nil
}
var value any
if artifact.Content != nil {
value = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
return execution.RecordEvidence("outcome.json", map[string]any{"outcome": outcome, "artifact": value, "error": errorText})
}
func captureCacheObservation(writer contracts.EvidenceWriter, observation contracts.CacheObservation) error {
if writer == nil {
return nil
}
errorText := ""
if observation.Err != nil {
errorText = observation.Err.Error()
}
if err := writer.WriteJSON("cache/"+observation.Decision+".json", map[string]any{
"decision": observation.Decision, "message_content_hash": observation.MessageContentHash,
"error": errorText,
}); err != nil {
return &contracts.CaptureError{Cause: err}
}
if observation.Artifact != nil {
if err := writer.WriteBytes("cache/"+observation.Decision+"-artifact.json", observation.Artifact); err != nil {
return &contracts.CaptureError{Cause: fmt.Errorf("record cached artifact: %w", err)}
}
}
return nil
}
func assessmentEvidence(assessment contracts.CandidateAssessment) []any {
findings := make([]any, 0, len(assessment.Validators))
for _, finding := range assessment.Validators {
errorText := ""
if finding.Failure != nil {
errorText = finding.Failure.Error()
}
findings = append(findings, map[string]any{"validator": finding.ValidatorName, "decision": finding.Decision,
"result": finding.Result, "correction_guidance": finding.Result.CorrectionGuidance, "error": errorText})
}
return findings
}

View File

@@ -0,0 +1,177 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
func (*nativeEvidence) Enabled() bool { return true }
func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) {
evidence := &nativeEvidence{}
text := strings.Repeat("sk-ordinary-source-content ", 500)
reference := strings.Repeat("standalone-reference-content ", 500)
doc := typedTestDocument()
doc.Units[0].Text = text
incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false)
if err != nil {
t.Fatal(err)
}
target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"),
chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref},
references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}},
}
binding := Binding("producer")
binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"}
if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil {
t.Fatal(err)
}
content := string(evidence.files["inputs.json"])
for _, value := range []string{text, reference, "chunk material", "incoming-value"} {
if !strings.Contains(content, value) {
t.Fatalf("missing self-contained input %q", value)
}
}
if strings.Contains(content, "configured-option-must-use-native-inspection") {
t.Fatal("runtime configuration bypassed secret-aware inspection")
}
}
func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
for _, reuse := range []bool{false, true} {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1})
if err != nil || output.FallbackCount != 0 || calls != 1 {
t.Fatalf("run: %#v %v, calls=%d", output, err, calls)
}
if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 {
t.Fatal("missing chunk/extract/merge/normalization boundaries")
}
if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] {
t.Fatal("cache reuse fabricated a native execution")
}
for name, content := range evidence.files {
if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") {
continue
}
var outcome struct {
Artifact struct{ Content debugBinaryEnvelope }
}
if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) {
t.Fatalf("missing codec artifact for %s: %s %v", name, content, err)
}
}
if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 {
t.Fatal("reused artifact bytes not retained")
}
var snapshot promptkit.ExecutionGroupSummary
if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil {
t.Fatal(err)
}
if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete {
t.Fatalf("incorrect current-run totals: %#v", snapshot)
}
if len(evidence.files["composition.json"]) == 0 {
t.Fatal("resolved composition unavailable")
}
}
}
func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) {
for _, fail := range []bool{false, true} {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
probe := newRunnerProbe()
primary := errors.New("deterministic operation failed")
producer := runnerNotesProducer{name: "notes", probe: probe}
if fail {
producer.failure = primary
}
lane := registerRunnerNotes(t, catalog, producer)
permissive := contracts.ModePermissive
lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe)
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
evidence := &nativeEvidence{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1})
if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) {
t.Fatalf("result=%#v %v", output, err)
}
found := false
for name, content := range evidence.files {
if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") {
continue
}
found = true
var result struct {
Outcome contracts.ProducerOutcome
Error string
Artifact any
}
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
if fail {
if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil {
t.Fatalf("fabricated failed artifact: %s", content)
}
} else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil {
t.Fatalf("fallback lost qualification: %s", content)
}
}
if !found {
t.Fatal("failed/fallback operation not captured")
}
}
}
func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) {
primary := errors.New("interpretation evidence failed")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
fallbacks := 0
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) {
return nil, nil, &contracts.CaptureError{Cause: primary}
}, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks)
}
}
func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) {
binding := Binding("configured-module")
binding.Options = map[string]any{"token": "configuration-secret"}
validator := Binding("configured-validator")
validator.Options = map[string]any{"token": "validator-configuration-secret"}
binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}}
resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}}
content, err := json.Marshal(captureComposition(resolved))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") {
t.Fatalf("composition lost identities or bypassed native inspection: %s", content)
}
if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" {
t.Fatal("capture mutated configured module options")
}
}

View File

@@ -44,6 +44,22 @@ func RunProducer[T any](
accepted := false
evaluate := func(result *promptkit.RunResult) {
candidate, guidance, failure, accepted = nil, nil, nil, false
var assessment contracts.CandidateAssessment
defer func() {
if operation.Execution.Capture == nil || result == nil {
return
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
if err := operation.Execution.RecordEvidence("interpretations/"+result.RunID+".json", map[string]any{
"run_id": result.RunID, "usable_candidate": candidate != nil, "accepted": accepted,
"correction_guidance": guidance, "assessment": assessmentEvidence(assessment), "error": errorText,
}); err != nil {
assessmentError = errors.Join(assessmentError, err)
}
}()
if canceled := ctx.Err(); canceled != nil {
failure = canceled
return
@@ -58,6 +74,10 @@ func RunProducer[T any](
return
}
if failure != nil {
var captureErr *contracts.CaptureError
if errors.As(failure, &captureErr) {
assessmentError = failure
}
candidate, guidance = nil, nil
return
}
@@ -65,7 +85,8 @@ func RunProducer[T any](
failure = errors.New("producer response could not be interpreted as a usable candidate")
return
}
assessment, err := operation.Assess(ctx, *candidate)
var err error
assessment, err = operation.Assess(ctx, *candidate)
if err != nil {
assessmentError = err
return
@@ -85,10 +106,31 @@ func RunProducer[T any](
request := producerRequest(initial, operation.Execution, true)
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
request, capture := CaptureNativeRequest(request, scopeEvidence(operation.Execution.Capture, "initial"))
if captureErr := capture.Err(); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
var result *promptkit.RunResult
var err error
var cacheKey string
cache := operation.Cache
if captureErr := operation.Execution.RecordEvidence("cache/selection.json", map[string]any{"mode": operation.Controls.Cache, "reuse_eligible": operation.ReuseEligible, "store_available": cache != nil && cache.Store != nil}); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
if cache != nil && operation.Execution.Capture != nil {
owned := *cache
existing := owned.Observe
owned.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
if err := captureCacheObservation(operation.Execution.Capture, observation); err != nil {
return err
}
if existing != nil {
return existing(ctx, observation)
}
return nil
}
cache = &owned
}
if cache != nil && cache.Store != nil && operation.ReuseEligible && operation.Controls.Cache != contracts.CacheBypass {
if cache.Encode == nil || cache.Decode == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer cache codec must be supplied")
@@ -101,15 +143,21 @@ func RunProducer[T any](
if err == nil {
defer prepared.Discard()
details := prepared.Details()
if capture != nil {
capture.RetainPrepared(copyPreparedEvidence(details))
}
cacheKey = details.MessageContentHash
if captureErr := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "prepared", MessageContentHash: cacheKey, Prepared: &details}); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
return zero, contracts.ProducerOutcome{}, captureFailure(captureErr, capture.Finish(nil, captureErr, false, false))
}
reused, lookupErr := lookupProducerCache(ctx, cache, operation.Controls.Cache, cacheKey)
if lookupErr != nil {
return zero, contracts.ProducerOutcome{}, lookupErr
return zero, contracts.ProducerOutcome{}, captureFailure(lookupErr, capture.Finish(nil, lookupErr, false, false))
}
if reused != nil {
if captureErr := capture.Finish(nil, nil, false, true); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
return *reused, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}, nil
}
result, err = engine.RunPrepared(ctx, prepared)
@@ -121,11 +169,14 @@ func RunProducer[T any](
decision = "ineligible"
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: decision}); err != nil {
return zero, contracts.ProducerOutcome{}, err
return zero, contracts.ProducerOutcome{}, captureFailure(err, capture.Finish(nil, err, false, false))
}
}
result, err = engine.Run(ctx, request)
}
if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureFailure(errors.Join(ctx.Err(), err), captureErr)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
@@ -142,11 +193,18 @@ func RunProducer[T any](
}
if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 {
var pendingCapture *NativeCapture
var recordingErr error
coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer,
Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
// The first callback has no new result. Its findings already exist.
if turn.Result != nil {
recordingErr = pendingCapture.Finish(turn.Result, nil, false, false)
pendingCapture = nil
if recordingErr != nil {
return nil, recordingErr
}
evaluate(turn.Result)
}
if assessmentError != nil {
@@ -158,8 +216,19 @@ func RunProducer[T any](
// Once another execution is selected, an earlier candidate is not a
// terminal fallback source if construction or native execution fails.
candidate = nil
correctionPath := ""
if operation.Execution.Capture != nil {
correctionPath = "corrections/" + turn.Exchange.Details().RunID
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/assignment.json", map[string]any{"guidance": guidance})
if recordingErr != nil {
return nil, recordingErr
}
}
next, buildErr := correct(ctx, guidance)
if buildErr != nil {
if correctionPath != "" {
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/construction-failure.json", map[string]any{"error": buildErr.Error()})
}
return nil, buildErr
}
if initial.ProfileID != "" {
@@ -169,9 +238,21 @@ func RunProducer[T any](
next.SessionID = initial.SessionID
}
next = producerRequest(next, operation.Execution, false)
if operation.Execution.Capture != nil {
next, pendingCapture = CaptureNativeRequest(next, scopeEvidence(operation.Execution.Capture, correctionPath))
if recordingErr = pendingCapture.Err(); recordingErr != nil {
return nil, recordingErr
}
}
return &next, nil
},
})
if pendingCapture != nil {
recordingErr = errors.Join(recordingErr, pendingCapture.Finish(coordination.Undelivered, coordinateErr, coordination.Undelivered != nil, false))
}
if recordingErr != nil {
return zero, contracts.ProducerOutcome{}, captureFailure(coordinateErr, recordingErr)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}

View File

@@ -38,9 +38,9 @@ func lookupProducerCache[T any](ctx context.Context, cache *contracts.ProducerCa
return nil, canceled
}
if err != nil {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err})
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err, Artifact: content})
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash}); err != nil {
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash, Artifact: content}); err != nil {
return nil, err
}
return &value, nil
@@ -59,7 +59,7 @@ func storeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCac
if err := ctx.Err(); err != nil {
return err
}
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash}
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash, Artifact: content}
if err := cache.Store.Store(ctx, mode, hash, content); err != nil {
observation.Decision, observation.Err = "store_failed", err
}

View File

@@ -21,6 +21,10 @@ func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM
if cause == nil {
cause = errors.New("producer did not yield an accepted result")
}
var captureErr *contracts.CaptureError
if errors.As(cause, &captureErr) {
return zero, contracts.ProducerOutcome{}, cause
}
switch mode {
case contracts.ModeStrict:
return zero, contracts.ProducerOutcome{}, cause

View File

@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"mime"
"path"
@@ -14,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type Registries struct {
@@ -44,15 +46,18 @@ type RunInput struct {
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
// Executions is the invocation's native group. The runner enrolls its
// requests; callers may use Snapshot without maintaining copied totals.
Executions *promptkit.ExecutionGroup
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
Debug DebugRecorder
// Workers bounds ready extraction and lane-completion jobs together.
@@ -107,12 +112,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
debugRecorder = synchronizedDebugRecorder(debugRecorder)
input.Debug = debugRecorder
if input.Executions == nil {
input.Executions = &promptkit.ExecutionGroup{}
}
input.Execution.Observer = promptkit.ComposeObservers(input.Execution.Observer, input.Executions.Observe)
defer func() {
if debugRecorder.Enabled() {
if captureErr := debugRecorder.WriteJSON("executions.json", input.Executions.Snapshot()); captureErr != nil {
err = errors.Join(err, &contracts.CaptureError{Cause: captureErr})
output = failOutput(output)
}
}
}()
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
if diagnosticErr != nil {
return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr)
}
appendDiagnosticGroups(&output, inputDiagnostics)
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
if err := writeDebugTimed(debugRecorder, "pipeline.json", debugTimedEnvelope{
Stage: "run",
StartedAt: startedTime(input.StartedAt),
Payload: map[string]any{
@@ -123,6 +140,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}); err != nil {
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
}
if debugRecorder.Enabled() {
if err := debugRecorder.WriteJSON("composition.json", captureComposition(input.pipeline)); err != nil {
return failOutput(output), &contracts.CaptureError{Cause: err}
}
}
adapter := input.Prepared.input
var doc *source.SourceDocument
@@ -135,8 +157,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
SourceID: input.SourceID,
Path: input.Path,
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
Options: redactSensitiveMap(input.pipeline.Input.Options),
Metadata: redactSensitiveMap(input.Metadata),
Metadata: input.Metadata,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
@@ -184,8 +205,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"cache_mode": input.pipeline.ChunkControls.Cache,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
"metadata": redactSensitiveMap(input.Metadata),
"metadata": input.Metadata,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
@@ -282,8 +302,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
outputDebugPayload := map[string]any{
"normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs),
"options": redactSensitiveMap(input.pipeline.Output.Options),
"metadata": redactSensitiveMap(input.Metadata),
"metadata": input.Metadata,
}
if acceptedChunkMap != nil {
outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap})

View File

@@ -21,15 +21,32 @@ type chunkPlanExecution struct {
// Chunk planning remains serial before lane workers. Only a native chunk
// producer uses the accepted-artifact cache; deterministic plans run directly.
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (result chunkPlanExecution, err error) {
chunker := input.Prepared.chunker
result := chunkPlanExecution{}
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, origin, nil)
defer func() {
errorText := ""
if err != nil {
errorText = err.Error()
}
err = captureFailure(err, execution.RecordEvidence("outcome.json", map[string]any{"outcome": result.outcome, "plan": result.plan, "chunks": debugSourceChunkEnvelopes(result.chunks), "error": errorText}))
}()
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: execution}
if err := captureOperationInputs(execution, target, input.pipeline.Chunk, nil); err != nil {
return result, err
}
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
var assessment contracts.CandidateAssessment
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
candidateNumber := 0
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: execution, ReuseEligible: !input.reuseIneligible,
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
if execution.Capture != nil {
candidateNumber++
if err := execution.RecordEvidence(fmt.Sprintf("plans/candidate-%d.json", candidateNumber), plan); err != nil {
return contracts.CandidateAssessment{}, err
}
}
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {

View File

@@ -264,10 +264,10 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) {
}
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) {
state := job.lane
lane, typed := state.prepared.resolved, state.prepared.typed
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
if err := ctx.Err(); err != nil {
result.err = err
return result
@@ -283,10 +283,19 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
return result
}
references := operationReferenceSet(input, lane.ExtractReferences)
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
var capturedArtifact contracts.SerializedArtifact
defer func() {
result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err))
}()
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref}
if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil {
result.err = err
return result
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)
@@ -310,6 +319,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
capturedArtifact, result.outcome = stored, outcome
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
@@ -348,7 +358,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.S
reuseEligible: state.reuseEligible,
}
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {

View File

@@ -74,8 +74,7 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *sou
return nil
}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
state := mergeStageResult{}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (state mergeStageResult, err error) {
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.MergeReferences)
state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences)
@@ -90,10 +89,22 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
if err != nil {
return state, err
}
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
var outcome contracts.ProducerOutcome
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
if execution.Capture != nil {
incoming := make([]contracts.SerializedArtifact, len(extracts.accepted))
for index, value := range extracts.accepted {
incoming[index] = value.artifact
}
if err := captureOperationInputs(execution, target, lane.Merge, incoming); err != nil {
return state, err
}
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, prepared.mergeValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: execution, ReuseEligible: state.reuseEligible,
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)
@@ -130,8 +141,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
return state, nil
}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
state := normalizeStageResult{}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (state normalizeStageResult, err error) {
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.NormalizeReferences)
state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
@@ -142,10 +152,22 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *sou
if err != nil {
return state, err
}
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
var outcome contracts.ProducerOutcome
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
if execution.Capture != nil {
incoming, err := serializeArtifact(typed.codec, merged.Value, false)
if err != nil {
return state, fmt.Errorf("encode normalization input: %w", err)
}
if err := captureOperationInputs(execution, target, lane.Normalize, []contracts.SerializedArtifact{incoming}); err != nil {
return state, err
}
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, prepared.normalizeValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: execution, ReuseEligible: state.reuseEligible,
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)