Capture native execution facts and complete scoped debug evidence

This commit is contained in:
2026-11-22 03:19:43 +00:00
parent 8680060400
commit 9f66421f4f
24 changed files with 1688 additions and 543 deletions

View File

@@ -1440,6 +1440,34 @@ incremental recording failure, finalization failure, and no completion retarget.
## Stage 31: Attach Native Execution Facts And Complete Debug Evidence
**Status:** Complete. Runner invocations now enroll initial, coordinated, and
nested validator executions in one native ExecutionGroup, preserving native
identity, terminal coverage, and unknown usage without copied totals. Scoped
capture composes existing observers and lightweight preparation callbacks, reuses
split-preparation details on hits, and records native events, prepared contracts,
results, exact response text, and full readable exchanges. Undelivered responses
are explicitly unassessed; result-less and preparation failures retain availability
and error evidence. Capture failures preserve primary causes and cannot trigger
semantic retries or permissive fallback.
Added frozen secret-aware application/PromptKit configuration and source snapshots,
resolved composition, self-contained source/reference/incoming-artifact inputs,
codec candidates, complete findings/guidance, interpretation and correction inputs,
normalization originals/mappings/proposals, deterministic/fallback outcomes, and
cache decisions/artifact bytes. Native credential omissions and exact adopted
configuration sensitivity paths are preserved without scanning application text;
runtime options are not reserialized outside native inspection. Removed the old
completion decorator, synthetic IDs, duplicate accounting, and heuristic content
scrubbing. Evidence is written incrementally without whole-run content retention.
Focused native helper/runner/assessment tests, isolated real-kit configuration
capture tests, vet, repeated race checks, recorder/cache/contracts/generic/Seriatim/
reconciliation/asset tests, production framework/config builds, and diff checks
pass. Concrete D&D normalizer/validator and full CLI/pipeline suites remain blocked
at setup by legacy removed reporting-package imports and old ConfigKit's YAML
plugin/limit dependency; their runtime results, including the new normalization
capture regression, are not claimed passing. Scheduled cleanup and final verification
must execute them. Native run-handler composition and final reporting remain in
their assigned later stages.
**Goal:** use native observations/snapshots instead of parallel execution accounting.
**Inspect/change:** helper/validator request observers, runner operation boundaries,

View File

@@ -243,9 +243,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debugPath := capture.Path()
var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder()
if capture.Enabled() {
debugRecorder = capture
debugRecorder = newRunEvidenceRecorder(capture, loaded.Result())
}
if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil {
version, err := buildinfo.Version()
if err != nil {
return fail(err)
}
if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt, "selected_lanes": only, "llm_profile": llmProfile.value, "llm_session_id": requestedSessionID.value, "execution_overrides": runtimeOverrides}); err != nil {
return fail(err)
}
if err := captureRunConfiguration(capture, loaded.Result()); err != nil {
return fail(err)
}
@@ -299,7 +306,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))

157
internal/cli/run_capture.go Normal file
View File

@@ -0,0 +1,157 @@
package cli
import (
"bytes"
"fmt"
"maps"
"slices"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// captureRunConfiguration inspects the original, frozen application load. It
// neither reloads resources nor serializes typed settings containing credentials.
// Shared loaders remain independent of run storage.
func captureRunConfiguration(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) error {
if !recorder.Enabled() {
return nil
}
if result == nil {
return fmt.Errorf("capture configuration: application result is unavailable")
}
var application bytes.Buffer
if err := result.WriteEffective(&application); err != nil {
return fmt.Errorf("inspect application configuration: %w", err)
}
if err := recorder.WriteBytes("configuration/application.yml", application.Bytes()); err != nil {
return err
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
return fmt.Errorf("capture configuration: PromptKit inspection is unavailable")
}
var prompts bytes.Buffer
if err := view.WriteEffective(&prompts); err != nil {
return fmt.Errorf("inspect PromptKit configuration: %w", err)
}
if err := recorder.WriteBytes("configuration/promptkit.yml", prompts.Bytes()); err != nil {
return err
}
return recorder.WriteJSON("configuration/sources.json", result.Sources())
}
type runEvidenceRecorder struct {
*frameworkdebug.Recorder
profiles map[string][][]string
backends map[string][][]string
}
// Keep ConfigKit-marked provider settings omitted in native execution evidence,
// too. These are exact configuration paths from the adopted definition, not
// searches for credential-looking strings in application content.
func newRunEvidenceRecorder(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) pipeline.DebugRecorder {
writer := &runEvidenceRecorder{Recorder: recorder, profiles: map[string][][]string{}, backends: map[string][][]string{}}
if result == nil {
return writer
}
sensitive := config.Definition("").Sensitive
paths := result.Metadata().HistoricalPaths()
register := func(masks map[string][][]string, id string, prefix configkit.Path) {
for _, field := range paths {
if !strings.HasPrefix(field.String(), prefix.String()+"/") || !sensitive(field) {
continue
}
segments := strings.Split(strings.TrimPrefix(field.String(), prefix.String()+"/"), "/")
for index, segment := range segments {
segments[index] = strings.ReplaceAll(strings.ReplaceAll(segment, "~1", "/"), "~0", "~")
}
masks[id] = append(masks[id], segments)
}
}
root := configkit.Keys("promptkit")
for index, profile := range result.Config.PromptKit.Profiles {
register(writer.profiles, profile.ID, root.Key("profiles").Index(index).Key("extra_params"))
}
for index, backend := range result.Config.PromptKit.Backends {
register(writer.backends, backend.ID, root.Key("backends").Index(index).Key("extra_params"))
}
for namespace, collection := range result.Config.PromptKit.Collections {
prefix := root.Key("collections").Key(namespace)
for index, profile := range collection.Profiles {
register(writer.profiles, namespace+"/"+profile.ID, prefix.Key("profiles").Index(index).Key("extra_params"))
}
for index, backend := range collection.Backends {
register(writer.backends, namespace+"/"+backend.ID, prefix.Key("backends").Index(index).Key("extra_params"))
}
}
return writer
}
func (writer *runEvidenceRecorder) WriteJSON(name string, payload any) error {
switch value := payload.(type) {
case *promptkit.PreparedRun:
copy := *value
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
payload = copy
case *promptkit.RunResult:
copy := *value
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
payload = copy
}
return writer.Recorder.WriteJSON(name, payload)
}
func (writer *runEvidenceRecorder) target(target promptkit.ExecutionTarget, profile, backend string) promptkit.ExecutionTarget {
for _, segments := range writer.backends[backend] {
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
}
for _, segments := range writer.profiles[profile] {
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
}
return target
}
// Copy only changed map branches; never mutate the native snapshot delivered to
// another observer. Ordinary siblings and all application text remain untouched.
func omitEvidenceSetting(values map[string]any, segments []string) map[string]any {
if len(segments) == 0 || values == nil {
return values
}
value, found := values[segments[0]]
if !found {
return values
}
copy := maps.Clone(values)
if len(segments) == 1 {
copy[segments[0]] = "[REDACTED]"
return copy
}
copy[segments[0]] = omitEvidenceValue(value, segments[1:])
return copy
}
func omitEvidenceValue(value any, segments []string) any {
if len(segments) == 0 {
return "[REDACTED]"
}
switch value := value.(type) {
case map[string]any:
return omitEvidenceSetting(value, segments)
case []any:
index, err := strconv.Atoi(segments[0])
if err != nil || index < 0 || index >= len(value) {
return value
}
copy := slices.Clone(value)
copy[index] = omitEvidenceValue(copy[index], segments[1:])
return copy
default:
return value
}
}

View File

@@ -0,0 +1,128 @@
package cli
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6
promptkit:
profiles:
- id: fixture
model: sk-ordinary-model-name
extra_params: {token: captured-secret-value}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
// Mutating public typed settings must not change the frozen inspection.
mutated := "mutated-after-load"
result.Config.PromptKit.Profiles[0].Model = &mutated
recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(recorder, result); err != nil {
t.Fatal(err)
}
for _, name := range []string{"application.yml", "promptkit.yml"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") {
t.Fatalf("configuration capture lost frozen/native secret handling: %s", content)
}
}
if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(nil, nil); err != nil {
t.Fatalf("disabled capture inspected configuration: %v", err)
}
}
type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return client(ctx, request)
}
func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6
promptkit:
collections:
fixture:
profiles:
- id: base
endpoint: https://example.invalid/v1
model: sk-ordinary-model-name
api_key_required: false
extra_params:
token: configured-secret
items: [{authorization: nested-secret, caption: ordinary-caption}]
- id: chosen
extends: {id: base}
prompts:
- id: request
default_profile: fixture/chosen
messages: [{role: user, content: sk-ordinary-application-content}]
output: {format: text, validation_mode: none}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" {
t.Fatal("capture changed native execution")
}
return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil
})))
if err != nil {
t.Fatal(err)
}
recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result))
response, err := engine.Run(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if err := capture.Finish(response, nil, false, false); err != nil {
t.Fatal(err)
}
for _, name := range []string{"prepared.json", "result.json"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name))
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} {
if strings.Contains(string(content), secret) {
t.Fatalf("ConfigKit/native secret leaked in %s", name)
}
}
if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") {
t.Fatalf("ordinary native settings were rewritten: %s", content)
}
}
if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" {
t.Fatal("projection mutated the caller's native snapshot")
}
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt"))
if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") {
t.Fatalf("application content rewritten: %s %v", content, err)
}
}

View File

@@ -37,6 +37,7 @@ type CacheObservation struct {
Decision string
MessageContentHash string
Prepared *promptkit.PreparedRun
Artifact []byte `json:"-"`
Err error
ChunkID string
ChunkIndex *int
@@ -65,8 +66,35 @@ type ExecutionOptions struct {
TransientRetries *int
Observer promptkit.ExecutionObserver `json:"-"`
OnPrepared func(context.Context, promptkit.PreparedRun) `json:"-"`
Capture EvidenceWriter `json:"-"`
}
// EvidenceWriter receives optional operation-scoped debug evidence. Modules get
// no filesystem root and cannot use capture as an execution or reporting service.
type EvidenceWriter interface {
WriteJSON(string, any) error
WriteBytes(string, []byte) error
}
// RecordEvidence writes available application-owned evidence at an ordinary
// operation boundary. A recording failure must not become semantic fallback.
func (options ExecutionOptions) RecordEvidence(name string, value any) error {
if options.Capture == nil {
return nil
}
if err := options.Capture.WriteJSON(name, value); err != nil {
return &CaptureError{Cause: err}
}
return nil
}
// CaptureError is an ordinary framework failure, not a semantic assessment or a
// reason to invoke a model again. Its cause remains available to the caller.
type CaptureError struct{ Cause error }
func (e *CaptureError) Error() string { return "record debug evidence: " + e.Cause.Error() }
func (e *CaptureError) Unwrap() error { return e.Cause }
type AssessmentDecision string
const (

View File

@@ -6,7 +6,10 @@ import (
"errors"
"fmt"
"reflect"
"strconv"
"sync/atomic"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -30,6 +33,10 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i
}
validator.position = index + 1
result, err := invoke(ctx, validator)
var captureErr *contracts.CaptureError
if errors.As(err, &captureErr) {
return assessment, errors.Join(ctx.Err(), err)
}
if canceled := ctx.Err(); canceled != nil {
return assessment, canceled
}
@@ -70,6 +77,7 @@ func (e *assessmentFrameworkError) Unwrap() error { return e.cause }
// typedCandidateAssessor binds one operation's complete chain. Each invocation
// captures the new candidate once and decodes a detached view for each validator.
func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] {
var candidateNumber atomic.Uint64
return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
@@ -81,12 +89,21 @@ func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidat
if err != nil {
return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err)
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
if capture != nil {
if err := capture.WriteBytes("candidate.json", content); err != nil {
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
}
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
request.execution.Capture = validatorCapture
switch validator.resolved.Target {
case ValidatorTargetTyped:
request.value, err = codec.decodeCandidate(append([]byte(nil), content...))
@@ -153,6 +170,7 @@ func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument,
// chunkCandidateAssessor assesses materialized chunks, not raw model plans.
// The runner's plan callback owns materialization before this boundary.
func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] {
var candidateNumber atomic.Uint64
return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) {
if ctx == nil {
return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil")
@@ -167,12 +185,21 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
target := target
target.chunks = chunks
schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10))
if capture != nil {
if err := capture.WriteBytes("candidate.json", content); err != nil {
return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err}
}
}
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) {
validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module))
defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }()
request, err := isolatedValidationTarget(target)
if err != nil {
return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err}
}
request.llmProfile = validator.resolved.Binding.LLMProfile
request.execution.Capture = validatorCapture
switch validator.resolved.Target {
case ValidatorTargetChunk:
return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{
@@ -193,3 +220,17 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato
})
}
}
func captureValidationFinding(writer contracts.EvidenceWriter, finding contracts.ValidationResult, failure error) error {
if writer == nil {
return failure
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
if err := writer.WriteJSON("finding.json", map[string]any{"result": finding, "correction_guidance": finding.CorrectionGuidance, "error": errorText}); err != nil {
return errors.Join(failure, &contracts.CaptureError{Cause: err})
}
return failure
}

View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
@@ -124,6 +125,56 @@ func assessmentNotesCodec(t *testing.T) artifactCodecEntry {
return entry
}
func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) {
evidence := &nativeEvidence{}
content := strings.Repeat("sk-ordinary content ", 1000)
candidate := codecNotes{Items: []string{content}}
validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) {
if request.Execution.Capture == nil {
t.Fatal("validator has no scoped capture")
}
return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil
})
assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}})
for range 2 {
assessment, err := assess(context.Background(), candidate)
if err != nil || assessment.Accepted() {
t.Fatalf("assessment=%#v %v", assessment, err)
}
}
if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 {
t.Fatal("revised candidates or findings were overwritten")
}
for name, data := range evidence.files {
if strings.HasSuffix(name, "/candidate.json") {
decoded, err := assessmentNotesCodec(t).decodeCandidate(data)
if err != nil || !reflect.DeepEqual(decoded, candidate) {
t.Fatalf("codec candidate changed: %#v, %v", decoded, err)
}
}
if strings.HasSuffix(name, "/finding.json") {
var finding struct {
Guidance string `json:"correction_guidance"`
}
if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content {
t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err)
}
}
}
}
func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) {
primary := errors.New("capture failed")
called := 0
_, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) {
called++
return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary}
})
if called != 1 || !errors.Is(err, primary) {
t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err)
}
}
func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) {
candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}}
doc := typedTestDocument()

View File

@@ -1,22 +1,13 @@
package pipeline
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"strings"
"sync"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
@@ -36,21 +27,19 @@ func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
type debugTimedEnvelope struct {
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
Attempt int `json:"attempt,omitempty"`
AttemptKind string `json:"attempt_kind,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"`
Error string `json:"error,omitempty"`
Stage string `json:"stage,omitempty"`
StepID string `json:"step_id,omitempty"`
LaneID string `json:"lane_id,omitempty"`
ModuleKey string `json:"module_key,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
DurationMS int64 `json:"duration_ms"`
Payload any `json:"payload,omitempty"`
Error string `json:"error,omitempty"`
}
type debugBinaryEnvelope struct {
Text string `json:"text,omitempty"`
ContentBase64 string `json:"content_base64,omitempty"`
ContentDigest string `json:"content_digest,omitempty"`
MediaType string `json:"media_type,omitempty"`
@@ -61,7 +50,6 @@ type debugSourceInput struct {
SourceID string `json:"source_id,omitempty"`
Path string `json:"path,omitempty"`
Raw debugBinaryEnvelope `json:"raw,omitempty"`
Options map[string]any `json:"options,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
@@ -109,244 +97,6 @@ type debugSerializedOutput struct {
Content debugBinaryEnvelope `json:"content"`
}
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMPromptArtifact struct {
CallID string `json:"call_id"`
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
}
type debugLLMResponseArtifact struct {
CallID string `json:"call_id"`
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugValidationCall struct {
ValidatorName string `json:"validator_name"`
Request any `json:"request"`
Result contracts.ValidationResult `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
type debugLLMClient struct {
inner contracts.StructuredLLMClient
recorder DebugRecorder
mu sync.Mutex
counter int
}
type debugLLMScope struct {
prefix string
parent *debugLLMScope
mu sync.Mutex
calls []debugLLMCallReference
}
type debugLLMScopeContextKey struct{}
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
if client == nil || recorder == nil || !recorder.Enabled() {
return client
}
if _, ok := client.(*debugLLMClient); ok {
return client
}
return &debugLLMClient{inner: client, recorder: synchronizedDebugRecorder(recorder)}
}
// WithDebugLLMRecording decorates a shared LLM client so calls made by
// construction-injected modules participate in the run's debug recording.
func WithDebugLLMRecording(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
return wrapDebugLLMClient(client, recorder)
}
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.mu.Lock()
client.counter++
callID := fmt.Sprintf("%04d", client.counter)
client.mu.Unlock()
started := time.Now().UTC()
response, err := client.inner.CompleteStructured(ctx, req, out)
completed := time.Now().UTC()
errorText := ""
if err != nil {
errorText = err.Error()
}
scopePrefix := cleanDebugPath(req.StageName)
if req.StageName == "" {
scopePrefix = "llm"
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scopePrefix = scope.prefix
}
promptPath := ""
var writeErr error
if response.Debug != nil && response.Debug.Prompt != nil {
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMPromptArtifact{
CallID: callID,
Prompt: response.Debug.Prompt,
},
}))
}
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
responseMaterial := debugResponseMaterial(response)
fallbackMaterial := debugCompletionFallback(response)
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
writeErr = errors.Join(writeErr, contentErr)
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
Stage: req.StageName,
ModuleKey: req.StageName,
StartedAt: started,
CompletedAt: completed,
DurationMS: completed.Sub(started).Milliseconds(),
Payload: debugLLMResponseArtifact{
CallID: callID,
Response: responseForArtifact,
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
},
Error: errorText,
}))
callRef := debugLLMCallReference{
CallID: callID,
PromptPath: promptPath,
ResponsePath: responsePath,
ResponseContentPath: responseContentPath,
PromptID: req.PromptID,
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
}
if err != nil {
return response, err
}
if writeErr != nil {
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
}
return response, err
}
func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
scope := &debugLLMScope{
prefix: prefix,
parent: debugLLMScopeFromContext(ctx),
}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func withIsolatedDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
if ctx == nil {
ctx = context.Background()
}
scope := &debugLLMScope{prefix: prefix}
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
}
func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope {
if ctx == nil {
return nil
}
scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope)
return scope
}
func (scope *debugLLMScope) record(ref debugLLMCallReference) {
if scope == nil {
return
}
scope.mu.Lock()
scope.calls = append(scope.calls, ref)
scope.mu.Unlock()
if scope.parent != nil {
scope.parent.record(ref)
}
}
func (scope *debugLLMScope) references() []debugLLMCallReference {
if scope == nil {
return nil
}
scope.mu.Lock()
defer scope.mu.Unlock()
if len(scope.calls) == 0 {
return nil
}
out := make([]debugLLMCallReference, len(scope.calls))
copy(out, scope.calls)
return out
}
func cleanDebugPath(value string) string {
parts := strings.Split(value, "/")
out := make([]string, 0, len(parts))
for _, part := range parts {
out = append(out, fileio.EncodePathComponent(part))
}
return strings.Join(out, "/")
}
func debugFirstNonEmptyString(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
if recorder == nil || !recorder.Enabled() {
return nil
@@ -363,60 +113,17 @@ func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnv
return recorder.WriteJSON(name, envelope)
}
func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope {
if scope != nil {
envelope.LLMCalls = scope.references()
}
return envelope
}
func writeDebugAttempt(recorder DebugRecorder, attemptPath string, envelope debugTimedEnvelope, scope *debugLLMScope) error {
return writeDebugTimed(recorder, attemptPath+".json", debugEnvelopeWithLLMCalls(envelope, scope))
}
type attemptTerminalRecorder struct {
recorder DebugRecorder
path string
label string
scope *debugLLMScope
envelope debugTimedEnvelope
}
type attemptDebugPersistenceError struct {
label string
err error
}
func (e *attemptDebugPersistenceError) Error() string {
return fmt.Sprintf("write %s attempt debug artifact: %v", e.label, e.err)
}
func (e *attemptDebugPersistenceError) Unwrap() error { return e.err }
func newAttemptTerminalRecorder(recorder DebugRecorder, attemptPath, label string, scope *debugLLMScope, envelope debugTimedEnvelope) attemptTerminalRecorder {
return attemptTerminalRecorder{recorder: recorder, path: attemptPath, label: label, scope: scope, envelope: envelope}
}
func (r attemptTerminalRecorder) record(payload any, terminalErr error) error {
envelope := r.envelope
envelope.Payload = payload
if terminalErr != nil {
envelope.Error = terminalErr.Error()
}
if err := writeDebugAttempt(r.recorder, r.path, envelope, r.scope); err != nil {
debugErr := &attemptDebugPersistenceError{label: r.label, err: err}
return errors.Join(terminalErr, debugErr)
}
return terminalErr
}
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope {
content = redactSecretBytes(content)
text := ""
if utf8.Valid(content) {
text = string(content)
}
return debugBinaryEnvelope{
Text: text,
ContentBase64: base64.StdEncoding.EncodeToString(content),
ContentDigest: debugContentDigest(content),
MediaType: mediaType,
Metadata: redactSensitiveMap(metadata),
Metadata: metadata,
}
}
@@ -430,7 +137,7 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen
Format: doc.Format,
Digest: doc.Digest,
Units: cloneSourceUnitsForDebug(doc.Units),
Metadata: redactSensitiveMap(doc.Metadata),
Metadata: doc.Metadata,
}
}
@@ -442,7 +149,7 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk {
Ref: chunk.Ref,
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
Units: cloneSourceUnitsForDebug(chunk.Units),
Metadata: redactSensitiveMap(chunk.Metadata),
Metadata: chunk.Metadata,
Annotations: source.CloneChunkAnnotations(chunk.Annotations),
PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations),
}
@@ -457,9 +164,9 @@ func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit {
cloned[i] = source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: string(redactSecretBytes([]byte(unit.Text))),
Text: unit.Text,
Ref: unit.Ref,
Metadata: redactSensitiveMap(unit.Metadata),
Metadata: unit.Metadata,
}
}
return cloned
@@ -510,7 +217,6 @@ func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
digest := contracts.DigestArtifactSchema(schema)
schema.JSONSchema = nil
content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil)
content.ContentDigest = debugContentDigest(output.Artifact.Content)
return debugSerializedOutput{
@@ -552,189 +258,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
return out
}
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
return req.DebugSummary()
}
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
return debugStructuredCompletionResponse{
Content: string(response.Content),
Provider: response.Provider,
Model: response.Model,
ProfileID: response.ProfileID,
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
}
}
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
if response.Debug != nil && response.Debug.Response != nil {
return nil
}
fallback := debugCompletionResponse(response)
if fallback.Content == "" &&
fallback.Provider == "" &&
fallback.Model == "" &&
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil
}
return &fallback
}
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
responseCopy := cloneDebugResponseWithoutContent(response)
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
content := ""
if response != nil {
content = response.Content
}
if content == "" && fallback != nil {
content = fallback.Content
}
if content == "" {
return "", responseCopy, fallbackCopy, nil
}
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
if recorder == nil || !recorder.Enabled() {
return contentPath, responseCopy, fallbackCopy, nil
}
if err := recorder.WriteBytes(contentPath, data); err != nil {
return contentPath, responseCopy, fallbackCopy, err
}
return contentPath, responseCopy, fallbackCopy, nil
}
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
if response == nil {
return nil
}
clone := *response
clone.Content = ""
return &clone
}
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
if fallback == nil {
return nil
}
clone := *fallback
clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone
}
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
raw := []byte(content)
if json.Valid(raw) {
var formatted bytes.Buffer
if err := json.Indent(&formatted, raw, "", " "); err == nil {
formatted.WriteByte('\n')
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
}
}
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
}
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
if response.Debug == nil {
return nil
}
return response.Debug.Response
}
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
if response.Debug == nil || response.Debug.Response == nil {
return ""
}
return response.Debug.Response.ModelName
}
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
result.Message = string(redactSecretBytes([]byte(result.Message)))
result.CorrectionGuidance = ""
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
return result
}
func debugContentDigest(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
var secretPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
}
func redactSecretBytes(content []byte) []byte {
if len(content) == 0 || !utf8.Valid(content) {
return append([]byte(nil), content...)
}
text := string(content)
for _, pattern := range secretPatterns {
text = pattern.ReplaceAllString(text, "[REDACTED]")
}
return []byte(text)
}
func redactSensitiveMap(values map[string]any) map[string]any {
if len(values) == 0 {
return nil
}
out := make(map[string]any, len(values))
for key, value := range values {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
continue
}
out[key] = redactSensitiveValue(value)
}
return out
}
func redactSensitiveValue(value any) any {
switch typed := value.(type) {
case string:
return string(redactSecretBytes([]byte(typed)))
case map[string]any:
return redactSensitiveMap(typed)
case map[string]string:
out := make(map[string]string, len(typed))
for key, value := range typed {
if sensitiveKey(key) {
out[key] = "[REDACTED]"
} else {
out[key] = string(redactSecretBytes([]byte(value)))
}
}
return out
default:
return value
}
}
func sensitiveKey(key string) bool {
key = strings.ToLower(key)
return strings.Contains(key, "api_key") ||
strings.Contains(key, "apikey") ||
strings.Contains(key, "authorization") ||
strings.Contains(key, "bearer") ||
strings.Contains(key, "password") ||
strings.Contains(key, "secret") ||
strings.Contains(key, "token")
}

View File

@@ -0,0 +1,255 @@
package pipeline
import (
"context"
"errors"
"fmt"
"maps"
"path"
"strings"
"sync"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
// NativeCapture retains only one request's prepared snapshot. Events are written
// incrementally, not accumulated as a second execution/usage ledger. It never
// prepares or executes a request itself.
type NativeCapture struct {
writer contracts.EvidenceWriter
mu sync.Mutex
prepared *promptkit.PreparedRun
runID string
events uint64
err error
}
// CaptureNativeRequest composes existing hooks and enables native exchange
// capture only when evidence was requested. Request credentials remain excluded
// by the native request's own JSON representation.
func CaptureNativeRequest(request promptkit.RunRequest, writer contracts.EvidenceWriter) (promptkit.RunRequest, *NativeCapture) {
if writer == nil {
return request, nil
}
capture := &NativeCapture{writer: writer}
capture.remember(writer.WriteJSON("request.json", request))
request.CaptureExchange = true
existing := request.OnPrepared
request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) {
// No serialization or filesystem I/O while native admission is held.
// Native hands off a detached value. Copy mutable fields only when an
// existing callback also owns that value and may change it.
if existing != nil {
capture.RetainPrepared(copyPreparedEvidence(details))
existing(ctx, details)
} else {
capture.RetainPrepared(details)
}
}
request.Observer = promptkit.ComposeObservers(request.Observer, capture.observe)
return request, capture
}
// RetainPrepared uses the already available split-preparation details on cache
// paths, including hits on which OnPrepared and native events never occur.
func (capture *NativeCapture) RetainPrepared(details promptkit.PreparedRun) {
if capture == nil {
return
}
capture.mu.Lock()
defer capture.mu.Unlock()
capture.prepared = &details
}
func (capture *NativeCapture) remember(err error) {
if capture == nil || err == nil {
return
}
capture.mu.Lock()
defer capture.mu.Unlock()
if capture.err == nil {
capture.err = err
}
}
func (capture *NativeCapture) observe(event promptkit.ExecutionEvent) {
capture.mu.Lock()
defer capture.mu.Unlock()
capture.runID = event.RunID
capture.events++
err := capture.writer.WriteJSON(path.Join("native", event.RunID, "events", fmt.Sprintf("%06d.json", capture.events)), event)
if capture.err == nil {
capture.err = err
}
}
// Err exposes synchronous callback/write failures at the next normal boundary.
func (capture *NativeCapture) Err() error {
if capture == nil {
return nil
}
capture.mu.Lock()
defer capture.mu.Unlock()
if capture.err == nil {
return nil
}
return &contracts.CaptureError{Cause: capture.err}
}
// Finish records available evidence even when native execution returned no
// result. Unassessed marks a response withheld from the application by native
// coordination; cacheHit never fabricates an execution or historical result.
func (capture *NativeCapture) Finish(result *promptkit.RunResult, nativeErr error, unassessed, cacheHit bool) error {
if capture == nil {
return nil
}
capture.mu.Lock()
prepared, runID := capture.prepared, capture.runID
capture.prepared = nil
capture.mu.Unlock()
if result != nil {
runID = result.RunID
}
prefix := "unexecuted"
if runID != "" {
prefix = path.Join("native", runID)
}
writeJSON := func(name string, value any) {
capture.remember(capture.writer.WriteJSON(path.Join(prefix, name), value))
}
writeText := func(name, value string) {
capture.remember(capture.writer.WriteBytes(path.Join(prefix, name), []byte(value)))
}
if prepared != nil {
writeJSON("prepared.json", prepared)
writeText("prepared-messages.txt", readableNativeMessages("Prepared request messages", prepared.Messages))
}
if result != nil {
writeJSON("result.json", result)
writeText("response.txt", result.RawOutput)
if result.Exchange != nil {
details := result.Exchange.Details()
writeJSON("exchange.json", details)
writeText("messages.txt", readableNativeMessages("Completed exchange", details.Messages))
}
}
errorText := ""
if nativeErr != nil {
errorText = nativeErr.Error()
}
writeJSON("availability.json", map[string]any{
"run_id": runID, "prepared_available": prepared != nil,
"result_available": result != nil, "unassessed": unassessed,
"cache_hit": cacheHit, "error": errorText,
})
return capture.Err()
}
func readableNativeMessages(label string, messages []promptkit.RenderedMessage) string {
var text strings.Builder
text.WriteString(label + "\n")
for _, message := range messages {
text.WriteString("\n[" + message.Role + "]\n")
text.WriteString(message.Content)
text.WriteByte('\n')
}
return text.String()
}
type scopedEvidence struct {
writer contracts.EvidenceWriter
prefix string
}
func (scope scopedEvidence) WriteJSON(name string, value any) error {
return scope.writer.WriteJSON(path.Join(scope.prefix, name), value)
}
func (scope scopedEvidence) WriteBytes(name string, value []byte) error {
return scope.writer.WriteBytes(path.Join(scope.prefix, name), value)
}
func scopeEvidence(writer contracts.EvidenceWriter, prefix string) contracts.EvidenceWriter {
if writer == nil {
return nil
}
return scopedEvidence{writer: writer, prefix: prefix}
}
func operationExecution(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk) contracts.ExecutionOptions {
options := input.Execution
writer := options.Capture
if writer == nil && input.Debug != nil && input.Debug.Enabled() {
writer = input.Debug
}
prefix := "operations"
if origin.StepID != "" {
prefix = path.Join(prefix, "steps", fileio.EncodePathComponent(origin.StepID))
}
prefix = path.Join(prefix, string(origin.Stage))
if origin.LaneID != "" {
prefix = path.Join(prefix, "lanes", fileio.EncodePathComponent(origin.LaneID))
}
if chunk != nil {
prefix = path.Join(prefix, "chunks", fileio.EncodePathComponent(chunk.ID))
}
prefix = path.Join(prefix, "modules", fileio.EncodePathComponent(origin.ModuleKey))
options.Capture = scopeEvidence(writer, prefix)
return options
}
// Native schema/extra-parameter documents are JSON values. Copy their mutable
// containers, without serializing large message bodies in OnPrepared.
func copyEvidenceJSON(value any) any {
switch value := value.(type) {
case map[string]any:
copy := make(map[string]any, len(value))
for key, item := range value {
copy[key] = copyEvidenceJSON(item)
}
return copy
case []any:
copy := make([]any, len(value))
for index, item := range value {
copy[index] = copyEvidenceJSON(item)
}
return copy
default:
return value
}
}
func copyPreparedEvidence(details promptkit.PreparedRun) promptkit.PreparedRun {
details.InputHashes = maps.Clone(details.InputHashes)
if details.EffectiveModelParams.ExtraParams != nil {
details.EffectiveModelParams.ExtraParams = copyEvidenceJSON(details.EffectiveModelParams.ExtraParams).(map[string]any)
}
details.OutputContract.Schema = copyEvidenceJSON(details.OutputContract.Schema)
if details.StructuredOutput != nil {
structured := *details.StructuredOutput
if structured.JSONSchema != nil {
schema := *structured.JSONSchema
schema.Schema = copyEvidenceJSON(schema.Schema)
structured.JSONSchema = &schema
}
details.StructuredOutput = &structured
}
messages := make([]promptkit.RenderedMessage, len(details.Messages))
for index, message := range details.Messages {
messages[index] = message
if message.CacheControl != nil {
hint := *message.CacheControl
messages[index].CacheControl = &hint
}
}
details.Messages = messages
return details
}
func captureFailure(primary, recording error) error {
if recording == nil {
return primary
}
return errors.Join(primary, recording)
}

View File

@@ -0,0 +1,396 @@
package pipeline
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"sync"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type nativeEvidence struct {
mu sync.Mutex
files map[string][]byte
fail func(string) error
}
func (evidence *nativeEvidence) WriteJSON(name string, value any) error {
content, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return evidence.WriteBytes(name, content)
}
func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error {
evidence.mu.Lock()
defer evidence.mu.Unlock()
if evidence.fail != nil {
if err := evidence.fail(name); err != nil {
return err
}
}
if evidence.files == nil {
evidence.files = make(map[string][]byte)
}
evidence.files[name] = append([]byte(nil), content...)
return nil
}
func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int {
count := 0
for name := range evidence.files {
if strings.HasSuffix(name, suffix) {
count++
}
}
return count
}
func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
started, prepared, providerCalls := 0, 0, 0
initialExecutions := 0
source := strings.Repeat("sk-ordinary-application-text ", 1000)
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
providerCalls++
value := 1
if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") {
initialExecutions++
if initialExecutions == 1 {
value = 0
}
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil
}))
options := contracts.ExecutionOptions{
Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionStarted {
started++
}
}),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ },
Capture: evidence,
}
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options,
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
request, _ := producerCorrection(ctx, []string{"nested assessment"})
request = producerRequest(request, options, true)
request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value)))
result, err := engine.Run(ctx, request)
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
return contracts.CandidateAssessment{}, captureErr
}
if err != nil {
return contracts.CandidateAssessment{}, err
}
if value == 1 {
return acceptedProducerAssessment(ctx, value)
}
return rejectedProducerAssessment(ctx, value)
},
}
initial := nativeProducerInitial()
initial.APIKey = "request-credential-must-be-omitted"
initial.Inputs["source"] = promptkit.Inline(source)
initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) {
details.Messages[0].Content = "changed by existing preparation hook"
}
value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil)
if err != nil || value != 1 {
t.Fatalf("result=%d, %v", value, err)
}
summary := group.Snapshot()
if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete {
t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary)
}
if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete {
t.Fatalf("fabricated known usage: %#v", summary.Usage)
}
if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 {
t.Fatalf("missing native results/exchanges: %v", evidence.files)
}
var initialID string
var continuation promptkit.RunResult
for name, content := range evidence.files {
if strings.Contains(string(content), initial.APIKey) {
t.Fatalf("native credential leaked in %s", name)
}
if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") {
t.Fatalf("preparation hook mutated retained evidence: %s", name)
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) {
t.Fatal("selected source content was truncated or rewritten")
}
if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") {
var result promptkit.RunResult
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
initialID = result.RunID
}
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") {
if err := json.Unmarshal(content, &continuation); err != nil {
t.Fatal(err)
}
}
}
if initialID == "" || continuation.ParentRunID != initialID {
t.Fatalf("native parent identity lost: %q %#v", initialID, continuation)
}
if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") {
t.Fatalf("visible rejected exchange history lost: %q", got)
}
}
func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) {
_, store := filesystemProducerCache(t)
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
operation := numberCacheOperation(store)
if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil {
t.Fatal(err)
}
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe,
OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") },
}
operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) {
t.Fatal("cache hit reassessed")
return contracts.CandidateAssessment{}, nil
}
value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 {
t.Fatalf("hit=%d, %v", value, err)
}
if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 {
t.Fatal("cache hit fabricated execution/history")
}
var details promptkit.PreparedRun
if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" {
t.Fatalf("hit prepared details unavailable: %#v %v", details, err)
}
if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) {
t.Fatal("cache hit not labeled")
}
}
func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) {
for _, missing := range []bool{false, true} {
evidence := &nativeEvidence{}
primary := errors.New("provider stopped")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
request := nativeProducerInitial()
if missing {
request.PromptID = "not-present"
}
request, capture := CaptureNativeRequest(request, evidence)
result, err := engine.Run(context.Background(), request)
if err == nil || result != nil {
t.Fatalf("expected result-less native failure: %#v, %v", result, err)
}
if captureErr := capture.Finish(result, err, false, false); captureErr != nil {
t.Fatal(captureErr)
}
if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 {
t.Fatal("fabricated result/provider body")
}
if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] {
t.Fatal("prepared availability wrong")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") {
found = true
if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) {
t.Fatalf("failure evidence unavailable: %s", content)
}
}
}
if !found {
t.Fatal("missing failure evidence")
}
}
}
func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) {
evidence := &nativeEvidence{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
finished, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1},
Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
finished++
if finished == 2 {
cancel()
}
}
}},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls)
}
if nativeEvidenceCount(evidence, "/result.json") != 2 {
t.Fatal("undelivered native response lost")
}
found := false
for name, content := range evidence.files {
if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) {
found = true
}
}
if !found {
t.Fatal("undelivered response not labeled unassessed")
}
}
func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) {
for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} {
t.Run(boundary, func(t *testing.T) {
captureErr := errors.New("evidence write failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.Contains(name, boundary) {
return captureErr
}
return nil
}}
calls, assessed, fallbackCalls := 0, 0, 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence},
Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) {
assessed++
return rejectedProducerAssessment(ctx, value)
},
}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil })
var recording *contracts.CaptureError
if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls)
}
if boundary == "request.json" && calls != 0 {
t.Fatal("model ran after pre-execution capture failure")
}
})
}
}
func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
var prepared atomic.Int64
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
const operations = 12
failures := make(chan error, operations)
var workers sync.WaitGroup
for index := range operations {
workers.Go(func() {
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment,
Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)),
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) },
},
}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
failures <- err
})
}
workers.Wait()
close(failures)
for err := range failures {
if err != nil {
t.Fatal(err)
}
}
summary := group.Snapshot()
if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations {
t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json"))
}
}
func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) {
content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000)
metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"}
envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil)
decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64)
if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] {
t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err)
}
doc := typedTestDocument()
doc.Units[0].Text, doc.Metadata = content, metadata
if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 {
t.Fatal("parsed source content or ordinary metadata was rewritten")
}
schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)}
serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}})
if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) {
t.Fatal("available artifact schema was omitted")
}
}
func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) {
evidence := &nativeEvidence{}
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) {
return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil
}, nil)
if err == nil || calls != 1 {
t.Fatalf("missing correction preparation: %v calls=%d", err, calls)
}
found := false
for name, content := range evidence.files {
if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") {
found = true
}
}
if !found {
t.Fatal("constructed correction input lost before preparation")
}
}
func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) {
primary, recording := errors.New("provider failed"), errors.New("capture failed")
evidence := &nativeEvidence{fail: func(name string) error {
if strings.HasSuffix(name, "/prepared.json") {
return recording
}
return nil
}}
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, primary
}))
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil })
if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" {
t.Fatalf("failure causes lost: %#v %v", outcome, err)
}
}

View File

@@ -0,0 +1,115 @@
package pipeline
import (
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func captureOperationInputs(execution contracts.ExecutionOptions, target typedValidationTarget, binding ModuleBinding, incoming []contracts.SerializedArtifact) error {
if execution.Capture == nil {
return nil
}
references := make(map[string]any, len(target.references.Slots))
for name, slot := range target.references.Slots {
items := make([]any, 0, len(slot.Items))
for _, item := range slot.Items {
items = append(items, map[string]any{"reference": item, "content": debugContentEnvelope(item.Content, item.MediaType, nil, nil)})
}
references[name] = map[string]any{"declaration": slot.Slot, "items": items}
}
inputs := make([]debugSerializedOutput, len(incoming))
for index, artifact := range incoming {
inputs[index] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
}
var chunk any
if target.chunk != nil {
chunk = debugSourceChunkEnvelope(*target.chunk)
}
return execution.RecordEvidence("inputs.json", map[string]any{
"source": debugSourceDocumentEnvelope(target.source),
"source_input": debugContentEnvelope(target.sourceInput.Content, target.sourceInput.MediaType, nil, nil),
"chunk": chunk, "chunks": debugSourceChunkEnvelopes(target.chunks), "references": references,
"incoming_artifacts": inputs, "module": binding.Module, "llm_profile": binding.LLMProfile, "metadata": target.metadata,
})
}
// Configured options belong to ConfigKit's secret-aware snapshot, not another
// direct serialization of runtime configuration inside composition evidence.
func captureComposition(resolved ResolvedPipeline) ResolvedPipeline {
resolved = cloneResolvedPipeline(resolved)
resolved.Input, resolved.Chunk, resolved.Output = captureBinding(resolved.Input), captureBinding(resolved.Chunk), captureBinding(resolved.Output)
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
lane.Extract, lane.Merge, lane.Normalize = captureBinding(lane.Extract), captureBinding(lane.Merge), captureBinding(lane.Normalize)
for index := range lane.Validators {
lane.Validators[index] = captureBinding(lane.Validators[index])
}
}
}
for index := range resolved.ValidatorChains {
for validatorIndex := range resolved.ValidatorChains[index].Validators {
resolved.ValidatorChains[index].Validators[validatorIndex].Binding = captureBinding(resolved.ValidatorChains[index].Validators[validatorIndex].Binding)
}
}
return resolved
}
func captureBinding(binding ModuleBinding) ModuleBinding {
binding.Options = nil
for index := range binding.Validators.Validators {
binding.Validators.Validators[index] = captureBinding(binding.Validators.Validators[index])
}
return binding
}
func captureOperationResult(execution contracts.ExecutionOptions, artifact contracts.SerializedArtifact, outcome contracts.ProducerOutcome, failure error) error {
if execution.Capture == nil {
return nil
}
var value any
if artifact.Content != nil {
value = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact})
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
return execution.RecordEvidence("outcome.json", map[string]any{"outcome": outcome, "artifact": value, "error": errorText})
}
func captureCacheObservation(writer contracts.EvidenceWriter, observation contracts.CacheObservation) error {
if writer == nil {
return nil
}
errorText := ""
if observation.Err != nil {
errorText = observation.Err.Error()
}
if err := writer.WriteJSON("cache/"+observation.Decision+".json", map[string]any{
"decision": observation.Decision, "message_content_hash": observation.MessageContentHash,
"error": errorText,
}); err != nil {
return &contracts.CaptureError{Cause: err}
}
if observation.Artifact != nil {
if err := writer.WriteBytes("cache/"+observation.Decision+"-artifact.json", observation.Artifact); err != nil {
return &contracts.CaptureError{Cause: fmt.Errorf("record cached artifact: %w", err)}
}
}
return nil
}
func assessmentEvidence(assessment contracts.CandidateAssessment) []any {
findings := make([]any, 0, len(assessment.Validators))
for _, finding := range assessment.Validators {
errorText := ""
if finding.Failure != nil {
errorText = finding.Failure.Error()
}
findings = append(findings, map[string]any{"validator": finding.ValidatorName, "decision": finding.Decision,
"result": finding.Result, "correction_guidance": finding.Result.CorrectionGuidance, "error": errorText})
}
return findings
}

View File

@@ -0,0 +1,177 @@
package pipeline
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
func (*nativeEvidence) Enabled() bool { return true }
func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) {
evidence := &nativeEvidence{}
text := strings.Repeat("sk-ordinary-source-content ", 500)
reference := strings.Repeat("standalone-reference-content ", 500)
doc := typedTestDocument()
doc.Units[0].Text = text
incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false)
if err != nil {
t.Fatal(err)
}
target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"),
chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref},
references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}},
}
binding := Binding("producer")
binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"}
if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil {
t.Fatal(err)
}
content := string(evidence.files["inputs.json"])
for _, value := range []string{text, reference, "chunk material", "incoming-value"} {
if !strings.Contains(content, value) {
t.Fatalf("missing self-contained input %q", value)
}
}
if strings.Contains(content, "configured-option-must-use-native-inspection") {
t.Fatal("runtime configuration bypassed secret-aware inspection")
}
}
func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) {
_, store := filesystemProducerCache(t)
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})}
prepared := prepareRunnerProfile(t, catalog, profile, engine)
for _, reuse := range []bool{false, true} {
evidence := &nativeEvidence{}
var group promptkit.ExecutionGroup
output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1})
if err != nil || output.FallbackCount != 0 || calls != 1 {
t.Fatalf("run: %#v %v, calls=%d", output, err, calls)
}
if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 {
t.Fatal("missing chunk/extract/merge/normalization boundaries")
}
if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] {
t.Fatal("cache reuse fabricated a native execution")
}
for name, content := range evidence.files {
if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") {
continue
}
var outcome struct {
Artifact struct{ Content debugBinaryEnvelope }
}
if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) {
t.Fatalf("missing codec artifact for %s: %s %v", name, content, err)
}
}
if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 {
t.Fatal("reused artifact bytes not retained")
}
var snapshot promptkit.ExecutionGroupSummary
if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil {
t.Fatal(err)
}
if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete {
t.Fatalf("incorrect current-run totals: %#v", snapshot)
}
if len(evidence.files["composition.json"]) == 0 {
t.Fatal("resolved composition unavailable")
}
}
}
func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) {
for _, fail := range []bool{false, true} {
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
probe := newRunnerProbe()
primary := errors.New("deterministic operation failed")
producer := runnerNotesProducer{name: "notes", probe: probe}
if fail {
producer.failure = primary
}
lane := registerRunnerNotes(t, catalog, producer)
permissive := contracts.ModePermissive
lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe)
profile := typedResolutionProfile()
profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane}
evidence := &nativeEvidence{}
output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1})
if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) {
t.Fatalf("result=%#v %v", output, err)
}
found := false
for name, content := range evidence.files {
if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") {
continue
}
found = true
var result struct {
Outcome contracts.ProducerOutcome
Error string
Artifact any
}
if err := json.Unmarshal(content, &result); err != nil {
t.Fatal(err)
}
if fail {
if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil {
t.Fatalf("fabricated failed artifact: %s", content)
}
} else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil {
t.Fatalf("fallback lost qualification: %s", content)
}
}
if !found {
t.Fatal("failed/fallback operation not captured")
}
}
}
func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) {
primary := errors.New("interpretation evidence failed")
engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}))
fallbacks := 0
operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment}
_, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) {
return nil, nil, &contracts.CaptureError{Cause: primary}
}, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil })
if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" {
t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks)
}
}
func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) {
binding := Binding("configured-module")
binding.Options = map[string]any{"token": "configuration-secret"}
validator := Binding("configured-validator")
validator.Options = map[string]any{"token": "validator-configuration-secret"}
binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}}
resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}}
content, err := json.Marshal(captureComposition(resolved))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") {
t.Fatalf("composition lost identities or bypassed native inspection: %s", content)
}
if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" {
t.Fatal("capture mutated configured module options")
}
}

View File

@@ -44,6 +44,22 @@ func RunProducer[T any](
accepted := false
evaluate := func(result *promptkit.RunResult) {
candidate, guidance, failure, accepted = nil, nil, nil, false
var assessment contracts.CandidateAssessment
defer func() {
if operation.Execution.Capture == nil || result == nil {
return
}
errorText := ""
if failure != nil {
errorText = failure.Error()
}
if err := operation.Execution.RecordEvidence("interpretations/"+result.RunID+".json", map[string]any{
"run_id": result.RunID, "usable_candidate": candidate != nil, "accepted": accepted,
"correction_guidance": guidance, "assessment": assessmentEvidence(assessment), "error": errorText,
}); err != nil {
assessmentError = errors.Join(assessmentError, err)
}
}()
if canceled := ctx.Err(); canceled != nil {
failure = canceled
return
@@ -58,6 +74,10 @@ func RunProducer[T any](
return
}
if failure != nil {
var captureErr *contracts.CaptureError
if errors.As(failure, &captureErr) {
assessmentError = failure
}
candidate, guidance = nil, nil
return
}
@@ -65,7 +85,8 @@ func RunProducer[T any](
failure = errors.New("producer response could not be interpreted as a usable candidate")
return
}
assessment, err := operation.Assess(ctx, *candidate)
var err error
assessment, err = operation.Assess(ctx, *candidate)
if err != nil {
assessmentError = err
return
@@ -85,10 +106,31 @@ func RunProducer[T any](
request := producerRequest(initial, operation.Execution, true)
request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0)
request, capture := CaptureNativeRequest(request, scopeEvidence(operation.Execution.Capture, "initial"))
if captureErr := capture.Err(); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
var result *promptkit.RunResult
var err error
var cacheKey string
cache := operation.Cache
if captureErr := operation.Execution.RecordEvidence("cache/selection.json", map[string]any{"mode": operation.Controls.Cache, "reuse_eligible": operation.ReuseEligible, "store_available": cache != nil && cache.Store != nil}); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
if cache != nil && operation.Execution.Capture != nil {
owned := *cache
existing := owned.Observe
owned.Observe = func(ctx context.Context, observation contracts.CacheObservation) error {
if err := captureCacheObservation(operation.Execution.Capture, observation); err != nil {
return err
}
if existing != nil {
return existing(ctx, observation)
}
return nil
}
cache = &owned
}
if cache != nil && cache.Store != nil && operation.ReuseEligible && operation.Controls.Cache != contracts.CacheBypass {
if cache.Encode == nil || cache.Decode == nil {
return zero, contracts.ProducerOutcome{}, errors.New("producer cache codec must be supplied")
@@ -101,15 +143,21 @@ func RunProducer[T any](
if err == nil {
defer prepared.Discard()
details := prepared.Details()
if capture != nil {
capture.RetainPrepared(copyPreparedEvidence(details))
}
cacheKey = details.MessageContentHash
if captureErr := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "prepared", MessageContentHash: cacheKey, Prepared: &details}); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
return zero, contracts.ProducerOutcome{}, captureFailure(captureErr, capture.Finish(nil, captureErr, false, false))
}
reused, lookupErr := lookupProducerCache(ctx, cache, operation.Controls.Cache, cacheKey)
if lookupErr != nil {
return zero, contracts.ProducerOutcome{}, lookupErr
return zero, contracts.ProducerOutcome{}, captureFailure(lookupErr, capture.Finish(nil, lookupErr, false, false))
}
if reused != nil {
if captureErr := capture.Finish(nil, nil, false, true); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureErr
}
return *reused, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}, nil
}
result, err = engine.RunPrepared(ctx, prepared)
@@ -121,11 +169,14 @@ func RunProducer[T any](
decision = "ineligible"
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: decision}); err != nil {
return zero, contracts.ProducerOutcome{}, err
return zero, contracts.ProducerOutcome{}, captureFailure(err, capture.Finish(nil, err, false, false))
}
}
result, err = engine.Run(ctx, request)
}
if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil {
return zero, contracts.ProducerOutcome{}, captureFailure(errors.Join(ctx.Err(), err), captureErr)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}
@@ -142,11 +193,18 @@ func RunProducer[T any](
}
if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 {
var pendingCapture *NativeCapture
var recordingErr error
coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{
Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer,
Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
// The first callback has no new result. Its findings already exist.
if turn.Result != nil {
recordingErr = pendingCapture.Finish(turn.Result, nil, false, false)
pendingCapture = nil
if recordingErr != nil {
return nil, recordingErr
}
evaluate(turn.Result)
}
if assessmentError != nil {
@@ -158,8 +216,19 @@ func RunProducer[T any](
// Once another execution is selected, an earlier candidate is not a
// terminal fallback source if construction or native execution fails.
candidate = nil
correctionPath := ""
if operation.Execution.Capture != nil {
correctionPath = "corrections/" + turn.Exchange.Details().RunID
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/assignment.json", map[string]any{"guidance": guidance})
if recordingErr != nil {
return nil, recordingErr
}
}
next, buildErr := correct(ctx, guidance)
if buildErr != nil {
if correctionPath != "" {
recordingErr = operation.Execution.RecordEvidence(correctionPath+"/construction-failure.json", map[string]any{"error": buildErr.Error()})
}
return nil, buildErr
}
if initial.ProfileID != "" {
@@ -169,9 +238,21 @@ func RunProducer[T any](
next.SessionID = initial.SessionID
}
next = producerRequest(next, operation.Execution, false)
if operation.Execution.Capture != nil {
next, pendingCapture = CaptureNativeRequest(next, scopeEvidence(operation.Execution.Capture, correctionPath))
if recordingErr = pendingCapture.Err(); recordingErr != nil {
return nil, recordingErr
}
}
return &next, nil
},
})
if pendingCapture != nil {
recordingErr = errors.Join(recordingErr, pendingCapture.Finish(coordination.Undelivered, coordinateErr, coordination.Undelivered != nil, false))
}
if recordingErr != nil {
return zero, contracts.ProducerOutcome{}, captureFailure(coordinateErr, recordingErr)
}
if canceled := ctx.Err(); canceled != nil {
return zero, contracts.ProducerOutcome{}, canceled
}

View File

@@ -38,9 +38,9 @@ func lookupProducerCache[T any](ctx context.Context, cache *contracts.ProducerCa
return nil, canceled
}
if err != nil {
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err})
return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err, Artifact: content})
}
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash}); err != nil {
if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash, Artifact: content}); err != nil {
return nil, err
}
return &value, nil
@@ -59,7 +59,7 @@ func storeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCac
if err := ctx.Err(); err != nil {
return err
}
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash}
observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash, Artifact: content}
if err := cache.Store.Store(ctx, mode, hash, content); err != nil {
observation.Decision, observation.Err = "store_failed", err
}

View File

@@ -21,6 +21,10 @@ func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM
if cause == nil {
cause = errors.New("producer did not yield an accepted result")
}
var captureErr *contracts.CaptureError
if errors.As(cause, &captureErr) {
return zero, contracts.ProducerOutcome{}, cause
}
switch mode {
case contracts.ModeStrict:
return zero, contracts.ProducerOutcome{}, cause

View File

@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"mime"
"path"
@@ -14,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/promptkit"
)
type Registries struct {
@@ -44,15 +46,18 @@ type RunInput struct {
// on a hit. Shared observers must synchronize concurrent calls.
ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error
Execution contracts.ExecutionOptions
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
// Executions is the invocation's native group. The runner enrolls its
// requests; callers may use Snapshot without maintaining copied totals.
Executions *promptkit.ExecutionGroup
Prepared *PreparedPipeline
SourceID string
Path string
RawInput []byte
SessionID string
RunID string
StartedAt time.Time
Metadata map[string]any
Diagnostics []contracts.ProducerDiagnostic
Debug DebugRecorder
// Workers bounds ready extraction and lane-completion jobs together.
@@ -107,12 +112,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
debugRecorder = synchronizedDebugRecorder(debugRecorder)
input.Debug = debugRecorder
if input.Executions == nil {
input.Executions = &promptkit.ExecutionGroup{}
}
input.Execution.Observer = promptkit.ComposeObservers(input.Execution.Observer, input.Executions.Observe)
defer func() {
if debugRecorder.Enabled() {
if captureErr := debugRecorder.WriteJSON("executions.json", input.Executions.Snapshot()); captureErr != nil {
err = errors.Join(err, &contracts.CaptureError{Cause: captureErr})
output = failOutput(output)
}
}
}()
inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil)
if diagnosticErr != nil {
return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr)
}
appendDiagnosticGroups(&output, inputDiagnostics)
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
if err := writeDebugTimed(debugRecorder, "pipeline.json", debugTimedEnvelope{
Stage: "run",
StartedAt: startedTime(input.StartedAt),
Payload: map[string]any{
@@ -123,6 +140,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}); err != nil {
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
}
if debugRecorder.Enabled() {
if err := debugRecorder.WriteJSON("composition.json", captureComposition(input.pipeline)); err != nil {
return failOutput(output), &contracts.CaptureError{Cause: err}
}
}
adapter := input.Prepared.input
var doc *source.SourceDocument
@@ -135,8 +157,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
SourceID: input.SourceID,
Path: input.Path,
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
Options: redactSensitiveMap(input.pipeline.Input.Options),
Metadata: redactSensitiveMap(input.Metadata),
Metadata: input.Metadata,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
@@ -184,8 +205,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
"cache_mode": input.pipeline.ChunkControls.Cache,
"source": debugSourceDocumentEnvelope(doc),
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
"options": redactSensitiveMap(input.pipeline.Chunk.Options),
"metadata": redactSensitiveMap(input.Metadata),
"metadata": input.Metadata,
},
}); err != nil {
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
@@ -282,8 +302,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
}
outputDebugPayload := map[string]any{
"normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs),
"options": redactSensitiveMap(input.pipeline.Output.Options),
"metadata": redactSensitiveMap(input.Metadata),
"metadata": input.Metadata,
}
if acceptedChunkMap != nil {
outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap})

View File

@@ -21,15 +21,32 @@ type chunkPlanExecution struct {
// Chunk planning remains serial before lane workers. Only a native chunk
// producer uses the accepted-artifact cache; deterministic plans run directly.
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (result chunkPlanExecution, err error) {
chunker := input.Prepared.chunker
result := chunkPlanExecution{}
origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()}
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, origin, nil)
defer func() {
errorText := ""
if err != nil {
errorText = err.Error()
}
err = captureFailure(err, execution.RecordEvidence("outcome.json", map[string]any{"outcome": result.outcome, "plan": result.plan, "chunks": debugSourceChunkEnvelopes(result.chunks), "error": errorText}))
}()
target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: execution}
if err := captureOperationInputs(execution, target, input.pipeline.Chunk, nil); err != nil {
return result, err
}
assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators)
var assessment contracts.CandidateAssessment
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible,
candidateNumber := 0
operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: execution, ReuseEligible: !input.reuseIneligible,
Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) {
if execution.Capture != nil {
candidateNumber++
if err := execution.RecordEvidence(fmt.Sprintf("plans/candidate-%d.json", candidateNumber), plan); err != nil {
return contracts.CandidateAssessment{}, err
}
}
_, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
if err != nil {
if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic {

View File

@@ -264,10 +264,10 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) {
}
}
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult {
func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) {
state := job.lane
lane, typed := state.prepared.resolved, state.prepared.typed
result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index}
if err := ctx.Err(); err != nil {
result.err = err
return result
@@ -283,10 +283,19 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
return result
}
references := operationReferenceSet(input, lane.ExtractReferences)
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
var capturedArtifact contracts.SerializedArtifact
defer func() {
result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err))
}()
target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref}
if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil {
result.err = err
return result
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences),
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)
@@ -310,6 +319,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.
result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err)
return result
}
capturedArtifact, result.outcome = stored, outcome
result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk)
if err != nil {
result.err = err
@@ -348,7 +358,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.S
reuseEligible: state.reuseEligible,
}
local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...)
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil {
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil {
return local, &laneRunError{stage: StageExtract, err: err}
}
if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil {

View File

@@ -74,8 +74,7 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *sou
return nil
}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) {
state := mergeStageResult{}
func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (state mergeStageResult, err error) {
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.MergeReferences)
state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences)
@@ -90,10 +89,22 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
if err != nil {
return state, err
}
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil)
var outcome contracts.ProducerOutcome
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
if execution.Capture != nil {
incoming := make([]contracts.SerializedArtifact, len(extracts.accepted))
for index, value := range extracts.accepted {
incoming[index] = value.artifact
}
if err := captureOperationInputs(execution, target, lane.Merge, incoming); err != nil {
return state, err
}
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, prepared.mergeValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: execution, ReuseEligible: state.reuseEligible,
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)
@@ -130,8 +141,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.
return state, nil
}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) {
state := normalizeStageResult{}
func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (state normalizeStageResult, err error) {
lane, typed := prepared.resolved, prepared.typed
references := operationReferenceSet(input, lane.NormalizeReferences)
state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences)
@@ -142,10 +152,22 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *sou
if err != nil {
return state, err
}
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution}
execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil)
var outcome contracts.ProducerOutcome
defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }()
target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution}
if execution.Capture != nil {
incoming, err := serializeArtifact(typed.codec, merged.Value, false)
if err != nil {
return state, fmt.Errorf("encode normalization input: %w", err)
}
if err := captureOperationInputs(execution, target, lane.Normalize, []contracts.SerializedArtifact{incoming}); err != nil {
return state, err
}
}
var assessment contracts.CandidateAssessment
assess := typedCandidateAssessor[any](typed.codec, target, prepared.normalizeValidators)
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible,
operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: execution, ReuseEligible: state.reuseEligible,
Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) {
var err error
assessment, err = assess(ctx, value)

View File

@@ -2,13 +2,18 @@ package itemregistry
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity"
@@ -39,6 +44,43 @@ func itemNames(value dnd.ItemRegistry) []string {
return names
}
func TestNormalizationCaptureRetainsFixedMappingAndEveryProposal(t *testing.T) {
recorder, err := frameworkdebug.Start(t.TempDir(), "normalization", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
client := &recordingNormalizerClient{responses: []string{invalidItemProposal, safeFirstProposal}}
request := normalizeRequestWithSource(mergedItems(), semanticDocument())
request.Operation.Execution.Capture = recorder
request.Operation.Controls.MaxAdditionalTurns = 1
result, err := newNormalizer(t, client).Normalize(context.Background(), request)
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted {
t.Fatalf("normalization: %#v %v", result, err)
}
content, err := os.ReadFile(filepath.Join(recorder.Path(), "normalization", "originals.json"))
if err != nil {
t.Fatal(err)
}
var originals dnd.ItemRegistry
if err := json.Unmarshal(content, &originals); err != nil || len(originals.Items) != 3 {
t.Fatalf("fixed originals missing: %#v %v", originals, err)
}
content, err = os.ReadFile(filepath.Join(recorder.Path(), "normalization", "preparation.json"))
if err != nil {
t.Fatal(err)
}
var preparation struct {
Mappings []any `json:"candidate_mappings"`
}
if err := json.Unmarshal(content, &preparation); err != nil || len(preparation.Mappings) != 3 {
t.Fatalf("fixed handle mapping missing: %s %v", content, err)
}
proposals, err := os.ReadDir(filepath.Join(recorder.Path(), "normalization", "proposals"))
if err != nil || len(proposals) != 2 {
t.Fatalf("revised proposals lost: %v %v", proposals, err)
}
}
func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) {
finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}`
for _, test := range []struct {

View File

@@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
order := shared.NewSourceRefOrder(req.Source)
records, baseFindings := preprocessRecords(merged, order)
deterministic := recordList(records)
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
}
findings := append([]diagnostics.Finding(nil), baseFindings...)
var advisories []diagnostics.Finding
discardedGroups := -1
@@ -134,6 +137,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err != nil {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
}
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
}); err != nil {
return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err
}
switch preparation.Disposition() {
case semanticreconcile.InsufficientCandidates:
return noModel()
@@ -152,6 +161,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
return nil, nil, err
}
assessment := preparation.Assess(response)
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
return nil, nil, err
}
applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
if err != nil {
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)

View File

@@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
order := shared.NewSourceRefOrder(req.Source)
records, baseFindings := preprocessRecords(merged, order)
deterministic := recordList(records)
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
}
findings := append([]diagnostics.Finding(nil), baseFindings...)
discardedGroups := -1
var omissionDetails []string
@@ -133,6 +136,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err != nil {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
}
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
}); err != nil {
return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err
}
switch preparation.Disposition() {
case semanticreconcile.InsufficientCandidates:
return noModel()
@@ -151,6 +160,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
return nil, nil, err
}
assessment := preparation.Assess(response)
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
return nil, nil, err
}
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
if err != nil {
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)

View File

@@ -77,6 +77,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
order := shared.NewSourceRefOrder(req.Source)
records, baseFindings := preprocessRecords(merged, order)
deterministic := recordList(records)
if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
}
findings := append([]diagnostics.Finding(nil), baseFindings...)
discardedGroups := -1
var omissionDetails []string
@@ -132,6 +135,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
if err != nil {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err)
}
if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{
"disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(),
"candidate_mappings": preparation.CandidateMappings(), "candidates": candidates,
}); err != nil {
return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err
}
switch preparation.Disposition() {
case semanticreconcile.InsufficientCandidates:
return noModel()
@@ -150,6 +159,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize
return nil, nil, err
}
assessment := preparation.Assess(response)
if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil {
return nil, nil, err
}
applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order)
if err != nil {
return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err)

View File

@@ -3,6 +3,7 @@ package combat_semantics
import (
"context"
"encoding/json"
"errors"
"strings"
"go.maximumdirect.net/promptkit"
@@ -50,12 +51,19 @@ func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationR
}
material := shared.TranscriptPromptMaterial(sourceInput)
result, err := v.engine.Run(ctx, promptkit.RunRequest{
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{
PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))},
Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries,
Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared,
})
}, req.Execution.Capture)
if err := capture.Err(); err != nil {
return contracts.ValidationResult{}, err
}
result, err := v.engine.Run(ctx, request)
if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil {
return contracts.ValidationResult{}, errors.Join(ctx.Err(), err, captureErr)
}
if canceled := ctx.Err(); canceled != nil {
return contracts.ValidationResult{}, canceled
}