diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ce28755d..5ccc2e13 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1440,6 +1440,34 @@ incremental recording failure, finalization failure, and no completion retarget. ## Stage 31: Attach Native Execution Facts And Complete Debug Evidence +**Status:** Complete. Runner invocations now enroll initial, coordinated, and +nested validator executions in one native ExecutionGroup, preserving native +identity, terminal coverage, and unknown usage without copied totals. Scoped +capture composes existing observers and lightweight preparation callbacks, reuses +split-preparation details on hits, and records native events, prepared contracts, +results, exact response text, and full readable exchanges. Undelivered responses +are explicitly unassessed; result-less and preparation failures retain availability +and error evidence. Capture failures preserve primary causes and cannot trigger +semantic retries or permissive fallback. +Added frozen secret-aware application/PromptKit configuration and source snapshots, +resolved composition, self-contained source/reference/incoming-artifact inputs, +codec candidates, complete findings/guidance, interpretation and correction inputs, +normalization originals/mappings/proposals, deterministic/fallback outcomes, and +cache decisions/artifact bytes. Native credential omissions and exact adopted +configuration sensitivity paths are preserved without scanning application text; +runtime options are not reserialized outside native inspection. Removed the old +completion decorator, synthetic IDs, duplicate accounting, and heuristic content +scrubbing. Evidence is written incrementally without whole-run content retention. +Focused native helper/runner/assessment tests, isolated real-kit configuration +capture tests, vet, repeated race checks, recorder/cache/contracts/generic/Seriatim/ +reconciliation/asset tests, production framework/config builds, and diff checks +pass. Concrete D&D normalizer/validator and full CLI/pipeline suites remain blocked +at setup by legacy removed reporting-package imports and old ConfigKit's YAML +plugin/limit dependency; their runtime results, including the new normalization +capture regression, are not claimed passing. Scheduled cleanup and final verification +must execute them. Native run-handler composition and final reporting remain in +their assigned later stages. + **Goal:** use native observations/snapshots instead of parallel execution accounting. **Inspect/change:** helper/validator request observers, runner operation boundaries, diff --git a/internal/cli/run.go b/internal/cli/run.go index 5541f5cf..e26d32e7 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -243,9 +243,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i debugPath := capture.Path() var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder() if capture.Enabled() { - debugRecorder = capture + debugRecorder = newRunEvidenceRecorder(capture, loaded.Result()) } - if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil { + version, err := buildinfo.Version() + if err != nil { + return fail(err) + } + if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt, "selected_lanes": only, "llm_profile": llmProfile.value, "llm_session_id": requestedSessionID.value, "execution_overrides": runtimeOverrides}); err != nil { + return fail(err) + } + if err := captureRunConfiguration(capture, loaded.Result()); err != nil { return fail(err) } @@ -299,7 +306,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i if err != nil { return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err)) } - llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder) prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient}) if err != nil { return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err)) diff --git a/internal/cli/run_capture.go b/internal/cli/run_capture.go new file mode 100644 index 00000000..7a5b462b --- /dev/null +++ b/internal/cli/run_capture.go @@ -0,0 +1,157 @@ +package cli + +import ( + "bytes" + "fmt" + "maps" + "slices" + "strconv" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +// captureRunConfiguration inspects the original, frozen application load. It +// neither reloads resources nor serializes typed settings containing credentials. +// Shared loaders remain independent of run storage. +func captureRunConfiguration(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) error { + if !recorder.Enabled() { + return nil + } + if result == nil { + return fmt.Errorf("capture configuration: application result is unavailable") + } + var application bytes.Buffer + if err := result.WriteEffective(&application); err != nil { + return fmt.Errorf("inspect application configuration: %w", err) + } + if err := recorder.WriteBytes("configuration/application.yml", application.Bytes()); err != nil { + return err + } + view, ok := result.InspectSubtree(configkit.Keys("promptkit")) + if !ok { + return fmt.Errorf("capture configuration: PromptKit inspection is unavailable") + } + var prompts bytes.Buffer + if err := view.WriteEffective(&prompts); err != nil { + return fmt.Errorf("inspect PromptKit configuration: %w", err) + } + if err := recorder.WriteBytes("configuration/promptkit.yml", prompts.Bytes()); err != nil { + return err + } + return recorder.WriteJSON("configuration/sources.json", result.Sources()) +} + +type runEvidenceRecorder struct { + *frameworkdebug.Recorder + profiles map[string][][]string + backends map[string][][]string +} + +// Keep ConfigKit-marked provider settings omitted in native execution evidence, +// too. These are exact configuration paths from the adopted definition, not +// searches for credential-looking strings in application content. +func newRunEvidenceRecorder(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) pipeline.DebugRecorder { + writer := &runEvidenceRecorder{Recorder: recorder, profiles: map[string][][]string{}, backends: map[string][][]string{}} + if result == nil { + return writer + } + sensitive := config.Definition("").Sensitive + paths := result.Metadata().HistoricalPaths() + register := func(masks map[string][][]string, id string, prefix configkit.Path) { + for _, field := range paths { + if !strings.HasPrefix(field.String(), prefix.String()+"/") || !sensitive(field) { + continue + } + segments := strings.Split(strings.TrimPrefix(field.String(), prefix.String()+"/"), "/") + for index, segment := range segments { + segments[index] = strings.ReplaceAll(strings.ReplaceAll(segment, "~1", "/"), "~0", "~") + } + masks[id] = append(masks[id], segments) + } + } + root := configkit.Keys("promptkit") + for index, profile := range result.Config.PromptKit.Profiles { + register(writer.profiles, profile.ID, root.Key("profiles").Index(index).Key("extra_params")) + } + for index, backend := range result.Config.PromptKit.Backends { + register(writer.backends, backend.ID, root.Key("backends").Index(index).Key("extra_params")) + } + for namespace, collection := range result.Config.PromptKit.Collections { + prefix := root.Key("collections").Key(namespace) + for index, profile := range collection.Profiles { + register(writer.profiles, namespace+"/"+profile.ID, prefix.Key("profiles").Index(index).Key("extra_params")) + } + for index, backend := range collection.Backends { + register(writer.backends, namespace+"/"+backend.ID, prefix.Key("backends").Index(index).Key("extra_params")) + } + } + return writer +} + +func (writer *runEvidenceRecorder) WriteJSON(name string, payload any) error { + switch value := payload.(type) { + case *promptkit.PreparedRun: + copy := *value + copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID) + payload = copy + case *promptkit.RunResult: + copy := *value + copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID) + payload = copy + } + return writer.Recorder.WriteJSON(name, payload) +} + +func (writer *runEvidenceRecorder) target(target promptkit.ExecutionTarget, profile, backend string) promptkit.ExecutionTarget { + for _, segments := range writer.backends[backend] { + target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments) + } + for _, segments := range writer.profiles[profile] { + target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments) + } + return target +} + +// Copy only changed map branches; never mutate the native snapshot delivered to +// another observer. Ordinary siblings and all application text remain untouched. +func omitEvidenceSetting(values map[string]any, segments []string) map[string]any { + if len(segments) == 0 || values == nil { + return values + } + value, found := values[segments[0]] + if !found { + return values + } + copy := maps.Clone(values) + if len(segments) == 1 { + copy[segments[0]] = "[REDACTED]" + return copy + } + copy[segments[0]] = omitEvidenceValue(value, segments[1:]) + return copy +} + +func omitEvidenceValue(value any, segments []string) any { + if len(segments) == 0 { + return "[REDACTED]" + } + switch value := value.(type) { + case map[string]any: + return omitEvidenceSetting(value, segments) + case []any: + index, err := strconv.Atoi(segments[0]) + if err != nil || index < 0 || index >= len(value) { + return value + } + copy := slices.Clone(value) + copy[index] = omitEvidenceValue(copy[index], segments[1:]) + return copy + default: + return value + } +} diff --git a/internal/cli/run_capture_test.go b/internal/cli/run_capture_test.go new file mode 100644 index 00000000..4187dc31 --- /dev/null +++ b/internal/cli/run_capture_test.go @@ -0,0 +1,128 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) { + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6 +promptkit: + profiles: + - id: fixture + model: sk-ordinary-model-name + extra_params: {token: captured-secret-value} +`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())}) + if err != nil { + t.Fatal(err) + } + // Mutating public typed settings must not change the frozen inspection. + mutated := "mutated-after-load" + result.Config.PromptKit.Profiles[0].Model = &mutated + recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + if err := captureRunConfiguration(recorder, result); err != nil { + t.Fatal(err) + } + for _, name := range []string{"application.yml", "promptkit.yml"} { + content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") { + t.Fatalf("configuration capture lost frozen/native secret handling: %s", content) + } + } + if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil { + t.Fatal(err) + } + if err := captureRunConfiguration(nil, nil); err != nil { + t.Fatalf("disabled capture inspected configuration: %v", err) + } +} + +type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) + +func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return client(ctx, request) +} + +func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) { + result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6 +promptkit: + collections: + fixture: + profiles: + - id: base + endpoint: https://example.invalid/v1 + model: sk-ordinary-model-name + api_key_required: false + extra_params: + token: configured-secret + items: [{authorization: nested-secret, caption: ordinary-caption}] + - id: chosen + extends: {id: base} + prompts: + - id: request + default_profile: fixture/chosen + messages: [{role: user, content: sk-ordinary-application-content}] + output: {format: text, validation_mode: none} +`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())}) + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" { + t.Fatal("capture changed native execution") + } + return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil + }))) + if err != nil { + t.Fatal(err) + } + recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result)) + response, err := engine.Run(context.Background(), request) + if err != nil { + t.Fatal(err) + } + if err := capture.Finish(response, nil, false, false); err != nil { + t.Fatal(err) + } + for _, name := range []string{"prepared.json", "result.json"} { + content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} { + if strings.Contains(string(content), secret) { + t.Fatalf("ConfigKit/native secret leaked in %s", name) + } + } + if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") { + t.Fatalf("ordinary native settings were rewritten: %s", content) + } + } + if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" { + t.Fatal("projection mutated the caller's native snapshot") + } + content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt")) + if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") { + t.Fatalf("application content rewritten: %s %v", content, err) + } +} diff --git a/internal/framework/contracts/producer.go b/internal/framework/contracts/producer.go index cef67580..ff7ef516 100644 --- a/internal/framework/contracts/producer.go +++ b/internal/framework/contracts/producer.go @@ -37,6 +37,7 @@ type CacheObservation struct { Decision string MessageContentHash string Prepared *promptkit.PreparedRun + Artifact []byte `json:"-"` Err error ChunkID string ChunkIndex *int @@ -65,8 +66,35 @@ type ExecutionOptions struct { TransientRetries *int Observer promptkit.ExecutionObserver `json:"-"` OnPrepared func(context.Context, promptkit.PreparedRun) `json:"-"` + Capture EvidenceWriter `json:"-"` } +// EvidenceWriter receives optional operation-scoped debug evidence. Modules get +// no filesystem root and cannot use capture as an execution or reporting service. +type EvidenceWriter interface { + WriteJSON(string, any) error + WriteBytes(string, []byte) error +} + +// RecordEvidence writes available application-owned evidence at an ordinary +// operation boundary. A recording failure must not become semantic fallback. +func (options ExecutionOptions) RecordEvidence(name string, value any) error { + if options.Capture == nil { + return nil + } + if err := options.Capture.WriteJSON(name, value); err != nil { + return &CaptureError{Cause: err} + } + return nil +} + +// CaptureError is an ordinary framework failure, not a semantic assessment or a +// reason to invoke a model again. Its cause remains available to the caller. +type CaptureError struct{ Cause error } + +func (e *CaptureError) Error() string { return "record debug evidence: " + e.Cause.Error() } +func (e *CaptureError) Unwrap() error { return e.Cause } + type AssessmentDecision string const ( diff --git a/internal/framework/pipeline/candidate_assessment.go b/internal/framework/pipeline/candidate_assessment.go index 702d9be5..28b3e10c 100644 --- a/internal/framework/pipeline/candidate_assessment.go +++ b/internal/framework/pipeline/candidate_assessment.go @@ -6,7 +6,10 @@ import ( "errors" "fmt" "reflect" + "strconv" + "sync/atomic" + "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) @@ -30,6 +33,10 @@ func executeValidationChain(ctx context.Context, chain preparedValidatorChain, i } validator.position = index + 1 result, err := invoke(ctx, validator) + var captureErr *contracts.CaptureError + if errors.As(err, &captureErr) { + return assessment, errors.Join(ctx.Err(), err) + } if canceled := ctx.Err(); canceled != nil { return assessment, canceled } @@ -70,6 +77,7 @@ func (e *assessmentFrameworkError) Unwrap() error { return e.cause } // typedCandidateAssessor binds one operation's complete chain. Each invocation // captures the new candidate once and decodes a detached view for each validator. func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[T] { + var candidateNumber atomic.Uint64 return func(ctx context.Context, value T) (contracts.CandidateAssessment, error) { if ctx == nil { return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil") @@ -81,12 +89,21 @@ func typedCandidateAssessor[T any](codec artifactCodecEntry, target typedValidat if err != nil { return contracts.CandidateAssessment{}, fmt.Errorf("encode validation candidate: %w", err) } - return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10)) + if capture != nil { + if err := capture.WriteBytes("candidate.json", content); err != nil { + return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err} + } + } + return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) { + validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module)) + defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }() request, err := isolatedValidationTarget(target) if err != nil { return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err} } request.llmProfile = validator.resolved.Binding.LLMProfile + request.execution.Capture = validatorCapture switch validator.resolved.Target { case ValidatorTargetTyped: request.value, err = codec.decodeCandidate(append([]byte(nil), content...)) @@ -153,6 +170,7 @@ func cloneAssessmentSource(doc *source.SourceDocument) (*source.SourceDocument, // chunkCandidateAssessor assesses materialized chunks, not raw model plans. // The runner's plan callback owns materialization before this boundary. func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidatorChain) contracts.CandidateAssessor[[]source.Chunk] { + var candidateNumber atomic.Uint64 return func(ctx context.Context, chunks []source.Chunk) (contracts.CandidateAssessment, error) { if ctx == nil { return contracts.CandidateAssessment{}, errors.New("assessment context must not be nil") @@ -167,12 +185,21 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato target := target target.chunks = chunks schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)} - return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) { + capture := scopeEvidence(target.execution.Capture, "assessments/"+strconv.FormatUint(candidateNumber.Add(1), 10)) + if capture != nil { + if err := capture.WriteBytes("candidate.json", content); err != nil { + return contracts.CandidateAssessment{}, &contracts.CaptureError{Cause: err} + } + } + return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (finding contracts.ValidationResult, failure error) { + validatorCapture := scopeEvidence(capture, "validators/"+strconv.Itoa(validator.position)+"-"+fileio.EncodePathComponent(validator.resolved.Binding.Module)) + defer func() { failure = captureValidationFinding(validatorCapture, finding, failure) }() request, err := isolatedValidationTarget(target) if err != nil { return contracts.ValidationResult{}, &assessmentFrameworkError{cause: err} } request.llmProfile = validator.resolved.Binding.LLMProfile + request.execution.Capture = validatorCapture switch validator.resolved.Target { case ValidatorTargetChunk: return validator.chunk.Validate(ctx, contracts.ChunkValidationRequest{ @@ -193,3 +220,17 @@ func chunkCandidateAssessor(target typedValidationTarget, chain preparedValidato }) } } + +func captureValidationFinding(writer contracts.EvidenceWriter, finding contracts.ValidationResult, failure error) error { + if writer == nil { + return failure + } + errorText := "" + if failure != nil { + errorText = failure.Error() + } + if err := writer.WriteJSON("finding.json", map[string]any{"result": finding, "correction_guidance": finding.CorrectionGuidance, "error": errorText}); err != nil { + return errors.Join(failure, &contracts.CaptureError{Cause: err}) + } + return failure +} diff --git a/internal/framework/pipeline/candidate_assessment_test.go b/internal/framework/pipeline/candidate_assessment_test.go index 7f3a3276..e8184696 100644 --- a/internal/framework/pipeline/candidate_assessment_test.go +++ b/internal/framework/pipeline/candidate_assessment_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "reflect" + "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" @@ -124,6 +125,56 @@ func assessmentNotesCodec(t *testing.T) artifactCodecEntry { return entry } +func TestTypedAssessmentCapturesCodecCandidateAndFullScopedGuidance(t *testing.T) { + evidence := &nativeEvidence{} + content := strings.Repeat("sk-ordinary content ", 1000) + candidate := codecNotes{Items: []string{content}} + validator := assessmentNotesValidator(t, "domain", func(_ context.Context, request contracts.TypedValidationRequest[codecNotes]) (contracts.ValidationResult, error) { + if request.Execution.Capture == nil { + t.Fatal("validator has no scoped capture") + } + return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: content}, nil + }) + assess := typedCandidateAssessor[codecNotes](assessmentNotesCodec(t), typedValidationTarget{execution: contracts.ExecutionOptions{Capture: evidence}}, preparedValidatorChain{validators: []preparedValidator{validator}}) + for range 2 { + assessment, err := assess(context.Background(), candidate) + if err != nil || assessment.Accepted() { + t.Fatalf("assessment=%#v %v", assessment, err) + } + } + if nativeEvidenceCount(evidence, "/candidate.json") != 2 || nativeEvidenceCount(evidence, "/finding.json") != 2 { + t.Fatal("revised candidates or findings were overwritten") + } + for name, data := range evidence.files { + if strings.HasSuffix(name, "/candidate.json") { + decoded, err := assessmentNotesCodec(t).decodeCandidate(data) + if err != nil || !reflect.DeepEqual(decoded, candidate) { + t.Fatalf("codec candidate changed: %#v, %v", decoded, err) + } + } + if strings.HasSuffix(name, "/finding.json") { + var finding struct { + Guidance string `json:"correction_guidance"` + } + if err := json.Unmarshal(data, &finding); err != nil || finding.Guidance != content { + t.Fatalf("guidance was omitted, rewritten, or truncated: %#v %v", finding, err) + } + } + } +} + +func TestCandidateAssessmentCaptureFailureIsNotSemanticNonacceptance(t *testing.T) { + primary := errors.New("capture failed") + called := 0 + _, err := executeValidationChain(context.Background(), preparedValidatorChain{validators: []preparedValidator{{}, {}}}, func(context.Context, preparedValidator) (contracts.ValidationResult, error) { + called++ + return contracts.ValidationResult{}, &contracts.CaptureError{Cause: primary} + }) + if called != 1 || !errors.Is(err, primary) { + t.Fatalf("capture failure was treated as a validator failure: calls=%d, %v", called, err) + } +} + func TestTypedAssessmentIsolatesCandidateAndContextViews(t *testing.T) { candidate := codecNotes{Items: []string{"original"}, Labels: map[string]string{"label": "original"}, Details: &codecNoteDetails{Name: "original"}} doc := typedTestDocument() diff --git a/internal/framework/pipeline/debug.go b/internal/framework/pipeline/debug.go index 95552258..f427c6e2 100644 --- a/internal/framework/pipeline/debug.go +++ b/internal/framework/pipeline/debug.go @@ -1,22 +1,13 @@ package pipeline import ( - "bytes" - "context" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" - "errors" - "fmt" - "path" - "regexp" - "strings" - "sync" "time" "unicode/utf8" - "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) @@ -36,21 +27,19 @@ func (noopDebugRecorder) WriteJSON(string, any) error { return nil } func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil } type debugTimedEnvelope struct { - Stage string `json:"stage,omitempty"` - StepID string `json:"step_id,omitempty"` - LaneID string `json:"lane_id,omitempty"` - ModuleKey string `json:"module_key,omitempty"` - Attempt int `json:"attempt,omitempty"` - AttemptKind string `json:"attempt_kind,omitempty"` - StartedAt time.Time `json:"started_at"` - CompletedAt time.Time `json:"completed_at"` - DurationMS int64 `json:"duration_ms"` - Payload any `json:"payload,omitempty"` - LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"` - Error string `json:"error,omitempty"` + Stage string `json:"stage,omitempty"` + StepID string `json:"step_id,omitempty"` + LaneID string `json:"lane_id,omitempty"` + ModuleKey string `json:"module_key,omitempty"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + DurationMS int64 `json:"duration_ms"` + Payload any `json:"payload,omitempty"` + Error string `json:"error,omitempty"` } type debugBinaryEnvelope struct { + Text string `json:"text,omitempty"` ContentBase64 string `json:"content_base64,omitempty"` ContentDigest string `json:"content_digest,omitempty"` MediaType string `json:"media_type,omitempty"` @@ -61,7 +50,6 @@ type debugSourceInput struct { SourceID string `json:"source_id,omitempty"` Path string `json:"path,omitempty"` Raw debugBinaryEnvelope `json:"raw,omitempty"` - Options map[string]any `json:"options,omitempty"` Metadata map[string]any `json:"metadata,omitempty"` } @@ -109,244 +97,6 @@ type debugSerializedOutput struct { Content debugBinaryEnvelope `json:"content"` } -type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest - -type debugStructuredCompletionResponse struct { - Content string `json:"content,omitempty"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - ProfileID string `json:"profile_id,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` -} - -type debugLLMPromptArtifact struct { - CallID string `json:"call_id"` - Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"` -} - -type debugLLMResponseArtifact struct { - CallID string `json:"call_id"` - Response *contracts.LLMDebugResponse `json:"response,omitempty"` - Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"` - ContentPath string `json:"content_path,omitempty"` - Error string `json:"error,omitempty"` - Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` -} - -type debugLLMCallReference struct { - CallID string `json:"call_id"` - PromptPath string `json:"prompt_path,omitempty"` - ResponsePath string `json:"response_path"` - ResponseContentPath string `json:"response_content_path,omitempty"` - PromptID string `json:"prompt_id,omitempty"` - ProfileID string `json:"profile_id,omitempty"` - Model string `json:"model,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - RepairAttempts int `json:"repair_attempts,omitempty"` - Error bool `json:"error,omitempty"` - Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"` -} - -type debugValidationCall struct { - ValidatorName string `json:"validator_name"` - Request any `json:"request"` - Result contracts.ValidationResult `json:"result,omitempty"` - Error string `json:"error,omitempty"` -} - -type debugLLMClient struct { - inner contracts.StructuredLLMClient - recorder DebugRecorder - mu sync.Mutex - counter int -} - -type debugLLMScope struct { - prefix string - parent *debugLLMScope - mu sync.Mutex - calls []debugLLMCallReference -} - -type debugLLMScopeContextKey struct{} - -func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient { - if client == nil || recorder == nil || !recorder.Enabled() { - return client - } - if _, ok := client.(*debugLLMClient); ok { - return client - } - return &debugLLMClient{inner: client, recorder: synchronizedDebugRecorder(recorder)} -} - -// WithDebugLLMRecording decorates a shared LLM client so calls made by -// construction-injected modules participate in the run's debug recording. -func WithDebugLLMRecording(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient { - return wrapDebugLLMClient(client, recorder) -} - -func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.mu.Lock() - client.counter++ - callID := fmt.Sprintf("%04d", client.counter) - client.mu.Unlock() - - started := time.Now().UTC() - response, err := client.inner.CompleteStructured(ctx, req, out) - completed := time.Now().UTC() - errorText := "" - if err != nil { - errorText = err.Error() - } - - scopePrefix := cleanDebugPath(req.StageName) - if req.StageName == "" { - scopePrefix = "llm" - } - if scope := debugLLMScopeFromContext(ctx); scope != nil { - scopePrefix = scope.prefix - } - promptPath := "" - var writeErr error - if response.Debug != nil && response.Debug.Prompt != nil { - promptPath = path.Join(scopePrefix, "prompt-"+callID+".json") - writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{ - Stage: req.StageName, - ModuleKey: req.StageName, - StartedAt: started, - CompletedAt: completed, - DurationMS: completed.Sub(started).Milliseconds(), - Payload: debugLLMPromptArtifact{ - CallID: callID, - Prompt: response.Debug.Prompt, - }, - })) - } - responsePath := path.Join(scopePrefix, "response-"+callID+".json") - responseMaterial := debugResponseMaterial(response) - fallbackMaterial := debugCompletionFallback(response) - responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial) - writeErr = errors.Join(writeErr, contentErr) - writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{ - Stage: req.StageName, - ModuleKey: req.StageName, - StartedAt: started, - CompletedAt: completed, - DurationMS: completed.Sub(started).Milliseconds(), - Payload: debugLLMResponseArtifact{ - CallID: callID, - Response: responseForArtifact, - Fallback: fallbackForArtifact, - ContentPath: responseContentPath, - Error: errorText, - Execution: cloneLLMExecutionSummary(response.Execution), - }, - Error: errorText, - })) - callRef := debugLLMCallReference{ - CallID: callID, - PromptPath: promptPath, - ResponsePath: responsePath, - ResponseContentPath: responseContentPath, - PromptID: req.PromptID, - ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID), - Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)), - PromptTokens: response.PromptTokens, - CompletionTokens: response.CompletionTokens, - TotalTokens: response.TotalTokens, - RepairAttempts: response.RepairAttempts, - Error: err != nil, - Execution: cloneLLMExecutionSummary(response.Execution), - } - if scope := debugLLMScopeFromContext(ctx); scope != nil { - scope.record(callRef) - } - if err != nil { - return response, err - } - if writeErr != nil { - return response, fmt.Errorf("write LLM debug artifact: %w", writeErr) - } - return response, err -} - -func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) { - if ctx == nil { - ctx = context.Background() - } - scope := &debugLLMScope{ - prefix: prefix, - parent: debugLLMScopeFromContext(ctx), - } - return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope -} - -func withIsolatedDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) { - if ctx == nil { - ctx = context.Background() - } - scope := &debugLLMScope{prefix: prefix} - return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope -} - -func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope { - if ctx == nil { - return nil - } - scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope) - return scope -} - -func (scope *debugLLMScope) record(ref debugLLMCallReference) { - if scope == nil { - return - } - scope.mu.Lock() - scope.calls = append(scope.calls, ref) - scope.mu.Unlock() - if scope.parent != nil { - scope.parent.record(ref) - } -} - -func (scope *debugLLMScope) references() []debugLLMCallReference { - if scope == nil { - return nil - } - scope.mu.Lock() - defer scope.mu.Unlock() - if len(scope.calls) == 0 { - return nil - } - out := make([]debugLLMCallReference, len(scope.calls)) - copy(out, scope.calls) - return out -} - -func cleanDebugPath(value string) string { - parts := strings.Split(value, "/") - out := make([]string, 0, len(parts)) - for _, part := range parts { - out = append(out, fileio.EncodePathComponent(part)) - } - return strings.Join(out, "/") -} - -func debugFirstNonEmptyString(values ...string) string { - for _, value := range values { - if strings.TrimSpace(value) != "" { - return value - } - } - return "" -} - func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error { if recorder == nil || !recorder.Enabled() { return nil @@ -363,60 +113,17 @@ func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnv return recorder.WriteJSON(name, envelope) } -func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope { - if scope != nil { - envelope.LLMCalls = scope.references() - } - return envelope -} - -func writeDebugAttempt(recorder DebugRecorder, attemptPath string, envelope debugTimedEnvelope, scope *debugLLMScope) error { - return writeDebugTimed(recorder, attemptPath+".json", debugEnvelopeWithLLMCalls(envelope, scope)) -} - -type attemptTerminalRecorder struct { - recorder DebugRecorder - path string - label string - scope *debugLLMScope - envelope debugTimedEnvelope -} - -type attemptDebugPersistenceError struct { - label string - err error -} - -func (e *attemptDebugPersistenceError) Error() string { - return fmt.Sprintf("write %s attempt debug artifact: %v", e.label, e.err) -} - -func (e *attemptDebugPersistenceError) Unwrap() error { return e.err } - -func newAttemptTerminalRecorder(recorder DebugRecorder, attemptPath, label string, scope *debugLLMScope, envelope debugTimedEnvelope) attemptTerminalRecorder { - return attemptTerminalRecorder{recorder: recorder, path: attemptPath, label: label, scope: scope, envelope: envelope} -} - -func (r attemptTerminalRecorder) record(payload any, terminalErr error) error { - envelope := r.envelope - envelope.Payload = payload - if terminalErr != nil { - envelope.Error = terminalErr.Error() - } - if err := writeDebugAttempt(r.recorder, r.path, envelope, r.scope); err != nil { - debugErr := &attemptDebugPersistenceError{label: r.label, err: err} - return errors.Join(terminalErr, debugErr) - } - return terminalErr -} - func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, _ any) debugBinaryEnvelope { - content = redactSecretBytes(content) + text := "" + if utf8.Valid(content) { + text = string(content) + } return debugBinaryEnvelope{ + Text: text, ContentBase64: base64.StdEncoding.EncodeToString(content), ContentDigest: debugContentDigest(content), MediaType: mediaType, - Metadata: redactSensitiveMap(metadata), + Metadata: metadata, } } @@ -430,7 +137,7 @@ func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocumen Format: doc.Format, Digest: doc.Digest, Units: cloneSourceUnitsForDebug(doc.Units), - Metadata: redactSensitiveMap(doc.Metadata), + Metadata: doc.Metadata, } } @@ -442,7 +149,7 @@ func debugSourceChunkEnvelope(chunk source.Chunk) debugSourceChunk { Ref: chunk.Ref, Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil), Units: cloneSourceUnitsForDebug(chunk.Units), - Metadata: redactSensitiveMap(chunk.Metadata), + Metadata: chunk.Metadata, Annotations: source.CloneChunkAnnotations(chunk.Annotations), PlanAnnotations: source.CloneChunkAnnotations(chunk.PlanAnnotations), } @@ -457,9 +164,9 @@ func cloneSourceUnitsForDebug(units []source.SourceUnit) []source.SourceUnit { cloned[i] = source.SourceUnit{ ID: unit.ID, Kind: unit.Kind, - Text: string(redactSecretBytes([]byte(unit.Text))), + Text: unit.Text, Ref: unit.Ref, - Metadata: redactSensitiveMap(unit.Metadata), + Metadata: unit.Metadata, } } return cloned @@ -510,7 +217,6 @@ func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any { func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput { schema := contracts.CloneArtifactSchema(output.Artifact.Schema) digest := contracts.DigestArtifactSchema(schema) - schema.JSONSchema = nil content := debugContentEnvelope(output.Artifact.Content, output.Artifact.MediaType, output.Artifact.Metadata, nil) content.ContentDigest = debugContentDigest(output.Artifact.Content) return debugSerializedOutput{ @@ -552,189 +258,7 @@ func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile { return out } -func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest { - return req.DebugSummary() -} - -func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse { - return debugStructuredCompletionResponse{ - Content: string(response.Content), - Provider: response.Provider, - Model: response.Model, - ProfileID: response.ProfileID, - PromptTokens: response.PromptTokens, - CompletionTokens: response.CompletionTokens, - TotalTokens: response.TotalTokens, - Execution: cloneLLMExecutionSummary(response.Execution), - } -} - -func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse { - if response.Debug != nil && response.Debug.Response != nil { - return nil - } - fallback := debugCompletionResponse(response) - if fallback.Content == "" && - fallback.Provider == "" && - fallback.Model == "" && - fallback.ProfileID == "" && - fallback.PromptTokens == 0 && - fallback.CompletionTokens == 0 && - fallback.TotalTokens == 0 && - fallback.Execution == nil { - return nil - } - return &fallback -} - -func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) { - responseCopy := cloneDebugResponseWithoutContent(response) - fallbackCopy := cloneDebugFallbackWithoutContent(fallback) - content := "" - if response != nil { - content = response.Content - } - if content == "" && fallback != nil { - content = fallback.Content - } - if content == "" { - return "", responseCopy, fallbackCopy, nil - } - - contentPath, data := debugResponseContentFile(scopePrefix, callID, content) - if recorder == nil || !recorder.Enabled() { - return contentPath, responseCopy, fallbackCopy, nil - } - if err := recorder.WriteBytes(contentPath, data); err != nil { - return contentPath, responseCopy, fallbackCopy, err - } - return contentPath, responseCopy, fallbackCopy, nil -} - -func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse { - if response == nil { - return nil - } - clone := *response - clone.Content = "" - return &clone -} - -func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse { - if fallback == nil { - return nil - } - clone := *fallback - clone.Content = "" - clone.Execution = cloneLLMExecutionSummary(fallback.Execution) - return &clone -} - -func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary { - if summary == nil { - return nil - } - clone := *summary - return &clone -} - -func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) { - raw := []byte(content) - if json.Valid(raw) { - var formatted bytes.Buffer - if err := json.Indent(&formatted, raw, "", " "); err == nil { - formatted.WriteByte('\n') - return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes() - } - } - return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw -} - -func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse { - if response.Debug == nil { - return nil - } - return response.Debug.Response -} - -func debugResponseModel(response contracts.StructuredCompletionResponse) string { - if response.Debug == nil || response.Debug.Response == nil { - return "" - } - return response.Debug.Response.ModelName -} - -func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult { - result.Message = string(redactSecretBytes([]byte(result.Message))) - result.CorrectionGuidance = "" - result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath))) - return result -} - func debugContentDigest(content []byte) string { sum := sha256.Sum256(content) return "sha256:" + hex.EncodeToString(sum[:]) } - -var secretPatterns = []*regexp.Regexp{ - regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`), - regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`), -} - -func redactSecretBytes(content []byte) []byte { - if len(content) == 0 || !utf8.Valid(content) { - return append([]byte(nil), content...) - } - text := string(content) - for _, pattern := range secretPatterns { - text = pattern.ReplaceAllString(text, "[REDACTED]") - } - return []byte(text) -} - -func redactSensitiveMap(values map[string]any) map[string]any { - if len(values) == 0 { - return nil - } - out := make(map[string]any, len(values)) - for key, value := range values { - if sensitiveKey(key) { - out[key] = "[REDACTED]" - continue - } - out[key] = redactSensitiveValue(value) - } - return out -} - -func redactSensitiveValue(value any) any { - switch typed := value.(type) { - case string: - return string(redactSecretBytes([]byte(typed))) - case map[string]any: - return redactSensitiveMap(typed) - case map[string]string: - out := make(map[string]string, len(typed)) - for key, value := range typed { - if sensitiveKey(key) { - out[key] = "[REDACTED]" - } else { - out[key] = string(redactSecretBytes([]byte(value))) - } - } - return out - default: - return value - } -} - -func sensitiveKey(key string) bool { - key = strings.ToLower(key) - return strings.Contains(key, "api_key") || - strings.Contains(key, "apikey") || - strings.Contains(key, "authorization") || - strings.Contains(key, "bearer") || - strings.Contains(key, "password") || - strings.Contains(key, "secret") || - strings.Contains(key, "token") -} diff --git a/internal/framework/pipeline/native_capture.go b/internal/framework/pipeline/native_capture.go new file mode 100644 index 00000000..71a4e67c --- /dev/null +++ b/internal/framework/pipeline/native_capture.go @@ -0,0 +1,255 @@ +package pipeline + +import ( + "context" + "errors" + "fmt" + "maps" + "path" + "strings" + "sync" + + "gitea.maximumdirect.net/eric/notarius/internal/core/fileio" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +// NativeCapture retains only one request's prepared snapshot. Events are written +// incrementally, not accumulated as a second execution/usage ledger. It never +// prepares or executes a request itself. +type NativeCapture struct { + writer contracts.EvidenceWriter + mu sync.Mutex + prepared *promptkit.PreparedRun + runID string + events uint64 + err error +} + +// CaptureNativeRequest composes existing hooks and enables native exchange +// capture only when evidence was requested. Request credentials remain excluded +// by the native request's own JSON representation. +func CaptureNativeRequest(request promptkit.RunRequest, writer contracts.EvidenceWriter) (promptkit.RunRequest, *NativeCapture) { + if writer == nil { + return request, nil + } + capture := &NativeCapture{writer: writer} + capture.remember(writer.WriteJSON("request.json", request)) + request.CaptureExchange = true + existing := request.OnPrepared + request.OnPrepared = func(ctx context.Context, details promptkit.PreparedRun) { + // No serialization or filesystem I/O while native admission is held. + // Native hands off a detached value. Copy mutable fields only when an + // existing callback also owns that value and may change it. + if existing != nil { + capture.RetainPrepared(copyPreparedEvidence(details)) + existing(ctx, details) + } else { + capture.RetainPrepared(details) + } + } + request.Observer = promptkit.ComposeObservers(request.Observer, capture.observe) + return request, capture +} + +// RetainPrepared uses the already available split-preparation details on cache +// paths, including hits on which OnPrepared and native events never occur. +func (capture *NativeCapture) RetainPrepared(details promptkit.PreparedRun) { + if capture == nil { + return + } + capture.mu.Lock() + defer capture.mu.Unlock() + capture.prepared = &details +} + +func (capture *NativeCapture) remember(err error) { + if capture == nil || err == nil { + return + } + capture.mu.Lock() + defer capture.mu.Unlock() + if capture.err == nil { + capture.err = err + } +} + +func (capture *NativeCapture) observe(event promptkit.ExecutionEvent) { + capture.mu.Lock() + defer capture.mu.Unlock() + capture.runID = event.RunID + capture.events++ + err := capture.writer.WriteJSON(path.Join("native", event.RunID, "events", fmt.Sprintf("%06d.json", capture.events)), event) + if capture.err == nil { + capture.err = err + } +} + +// Err exposes synchronous callback/write failures at the next normal boundary. +func (capture *NativeCapture) Err() error { + if capture == nil { + return nil + } + capture.mu.Lock() + defer capture.mu.Unlock() + if capture.err == nil { + return nil + } + return &contracts.CaptureError{Cause: capture.err} +} + +// Finish records available evidence even when native execution returned no +// result. Unassessed marks a response withheld from the application by native +// coordination; cacheHit never fabricates an execution or historical result. +func (capture *NativeCapture) Finish(result *promptkit.RunResult, nativeErr error, unassessed, cacheHit bool) error { + if capture == nil { + return nil + } + capture.mu.Lock() + prepared, runID := capture.prepared, capture.runID + capture.prepared = nil + capture.mu.Unlock() + if result != nil { + runID = result.RunID + } + prefix := "unexecuted" + if runID != "" { + prefix = path.Join("native", runID) + } + writeJSON := func(name string, value any) { + capture.remember(capture.writer.WriteJSON(path.Join(prefix, name), value)) + } + writeText := func(name, value string) { + capture.remember(capture.writer.WriteBytes(path.Join(prefix, name), []byte(value))) + } + if prepared != nil { + writeJSON("prepared.json", prepared) + writeText("prepared-messages.txt", readableNativeMessages("Prepared request messages", prepared.Messages)) + } + if result != nil { + writeJSON("result.json", result) + writeText("response.txt", result.RawOutput) + if result.Exchange != nil { + details := result.Exchange.Details() + writeJSON("exchange.json", details) + writeText("messages.txt", readableNativeMessages("Completed exchange", details.Messages)) + } + } + errorText := "" + if nativeErr != nil { + errorText = nativeErr.Error() + } + writeJSON("availability.json", map[string]any{ + "run_id": runID, "prepared_available": prepared != nil, + "result_available": result != nil, "unassessed": unassessed, + "cache_hit": cacheHit, "error": errorText, + }) + return capture.Err() +} + +func readableNativeMessages(label string, messages []promptkit.RenderedMessage) string { + var text strings.Builder + text.WriteString(label + "\n") + for _, message := range messages { + text.WriteString("\n[" + message.Role + "]\n") + text.WriteString(message.Content) + text.WriteByte('\n') + } + return text.String() +} + +type scopedEvidence struct { + writer contracts.EvidenceWriter + prefix string +} + +func (scope scopedEvidence) WriteJSON(name string, value any) error { + return scope.writer.WriteJSON(path.Join(scope.prefix, name), value) +} +func (scope scopedEvidence) WriteBytes(name string, value []byte) error { + return scope.writer.WriteBytes(path.Join(scope.prefix, name), value) +} +func scopeEvidence(writer contracts.EvidenceWriter, prefix string) contracts.EvidenceWriter { + if writer == nil { + return nil + } + return scopedEvidence{writer: writer, prefix: prefix} +} + +func operationExecution(input RunInput, origin contracts.DiagnosticOrigin, chunk *source.Chunk) contracts.ExecutionOptions { + options := input.Execution + writer := options.Capture + if writer == nil && input.Debug != nil && input.Debug.Enabled() { + writer = input.Debug + } + prefix := "operations" + if origin.StepID != "" { + prefix = path.Join(prefix, "steps", fileio.EncodePathComponent(origin.StepID)) + } + prefix = path.Join(prefix, string(origin.Stage)) + if origin.LaneID != "" { + prefix = path.Join(prefix, "lanes", fileio.EncodePathComponent(origin.LaneID)) + } + if chunk != nil { + prefix = path.Join(prefix, "chunks", fileio.EncodePathComponent(chunk.ID)) + } + prefix = path.Join(prefix, "modules", fileio.EncodePathComponent(origin.ModuleKey)) + options.Capture = scopeEvidence(writer, prefix) + return options +} + +// Native schema/extra-parameter documents are JSON values. Copy their mutable +// containers, without serializing large message bodies in OnPrepared. +func copyEvidenceJSON(value any) any { + switch value := value.(type) { + case map[string]any: + copy := make(map[string]any, len(value)) + for key, item := range value { + copy[key] = copyEvidenceJSON(item) + } + return copy + case []any: + copy := make([]any, len(value)) + for index, item := range value { + copy[index] = copyEvidenceJSON(item) + } + return copy + default: + return value + } +} + +func copyPreparedEvidence(details promptkit.PreparedRun) promptkit.PreparedRun { + details.InputHashes = maps.Clone(details.InputHashes) + if details.EffectiveModelParams.ExtraParams != nil { + details.EffectiveModelParams.ExtraParams = copyEvidenceJSON(details.EffectiveModelParams.ExtraParams).(map[string]any) + } + details.OutputContract.Schema = copyEvidenceJSON(details.OutputContract.Schema) + if details.StructuredOutput != nil { + structured := *details.StructuredOutput + if structured.JSONSchema != nil { + schema := *structured.JSONSchema + schema.Schema = copyEvidenceJSON(schema.Schema) + structured.JSONSchema = &schema + } + details.StructuredOutput = &structured + } + messages := make([]promptkit.RenderedMessage, len(details.Messages)) + for index, message := range details.Messages { + messages[index] = message + if message.CacheControl != nil { + hint := *message.CacheControl + messages[index].CacheControl = &hint + } + } + details.Messages = messages + return details +} + +func captureFailure(primary, recording error) error { + if recording == nil { + return primary + } + return errors.Join(primary, recording) +} diff --git a/internal/framework/pipeline/native_capture_test.go b/internal/framework/pipeline/native_capture_test.go new file mode 100644 index 00000000..cfeedc1e --- /dev/null +++ b/internal/framework/pipeline/native_capture_test.go @@ -0,0 +1,396 @@ +package pipeline + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "sync" + "sync/atomic" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +type nativeEvidence struct { + mu sync.Mutex + files map[string][]byte + fail func(string) error +} + +func (evidence *nativeEvidence) WriteJSON(name string, value any) error { + content, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return evidence.WriteBytes(name, content) +} +func (evidence *nativeEvidence) WriteBytes(name string, content []byte) error { + evidence.mu.Lock() + defer evidence.mu.Unlock() + if evidence.fail != nil { + if err := evidence.fail(name); err != nil { + return err + } + } + if evidence.files == nil { + evidence.files = make(map[string][]byte) + } + evidence.files[name] = append([]byte(nil), content...) + return nil +} + +func nativeEvidenceCount(evidence *nativeEvidence, suffix string) int { + count := 0 + for name := range evidence.files { + if strings.HasSuffix(name, suffix) { + count++ + } + } + return count +} + +func TestNativeCapturePreservesFullEvidenceAndCountsEachExecutionOnce(t *testing.T) { + evidence := &nativeEvidence{} + var group promptkit.ExecutionGroup + started, prepared, providerCalls := 0, 0, 0 + initialExecutions := 0 + source := strings.Repeat("sk-ordinary-application-text ", 1000) + engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + providerCalls++ + value := 1 + if !strings.Contains(request.Prompt.Messages[len(request.Prompt.Messages)-1].Content, "nested assessment") { + initialExecutions++ + if initialExecutions == 1 { + value = 0 + } + } + return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%d}`, value)}, nil + })) + options := contracts.ExecutionOptions{ + Observer: promptkit.ComposeObservers(group.Observe, func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionStarted { + started++ + } + }), + OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared++ }, + Capture: evidence, + } + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Execution: options, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + request, _ := producerCorrection(ctx, []string{"nested assessment"}) + request = producerRequest(request, options, true) + request, capture := CaptureNativeRequest(request, scopeEvidence(evidence, fmt.Sprintf("validators/value-%d", value))) + result, err := engine.Run(ctx, request) + if captureErr := capture.Finish(result, err, false, false); captureErr != nil { + return contracts.CandidateAssessment{}, captureErr + } + if err != nil { + return contracts.CandidateAssessment{}, err + } + if value == 1 { + return acceptedProducerAssessment(ctx, value) + } + return rejectedProducerAssessment(ctx, value) + }, + } + initial := nativeProducerInitial() + initial.APIKey = "request-credential-must-be-omitted" + initial.Inputs["source"] = promptkit.Inline(source) + initial.OnPrepared = func(_ context.Context, details promptkit.PreparedRun) { + details.Messages[0].Content = "changed by existing preparation hook" + } + value, _, err := RunProducer(context.Background(), engine, operation, initial, interpretProducerNumber, producerCorrection, nil) + if err != nil || value != 1 { + t.Fatalf("result=%d, %v", value, err) + } + summary := group.Snapshot() + if started != 4 || prepared != 4 || providerCalls != 4 || len(summary.Members) != 4 || summary.Attempts != 4 || !summary.TerminalCoverageComplete { + t.Fatalf("duplicate or missing enrollment: started=%d prepared=%d calls=%d summary=%#v", started, prepared, providerCalls, summary) + } + if summary.Usage.TotalTokens.Known || summary.Usage.TotalTokens.Complete { + t.Fatalf("fabricated known usage: %#v", summary.Usage) + } + if nativeEvidenceCount(evidence, "/result.json") != 4 || nativeEvidenceCount(evidence, "/exchange.json") != 4 { + t.Fatalf("missing native results/exchanges: %v", evidence.files) + } + var initialID string + var continuation promptkit.RunResult + for name, content := range evidence.files { + if strings.Contains(string(content), initial.APIKey) { + t.Fatalf("native credential leaked in %s", name) + } + if strings.HasSuffix(name, "/prepared.json") && strings.Contains(string(content), "changed by existing preparation hook") { + t.Fatalf("preparation hook mutated retained evidence: %s", name) + } + if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/prepared-messages.txt") && !strings.Contains(string(content), source) { + t.Fatal("selected source content was truncated or rewritten") + } + if strings.HasPrefix(name, "initial/") && strings.HasSuffix(name, "/result.json") { + var result promptkit.RunResult + if err := json.Unmarshal(content, &result); err != nil { + t.Fatal(err) + } + initialID = result.RunID + } + if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/result.json") { + if err := json.Unmarshal(content, &continuation); err != nil { + t.Fatal(err) + } + } + } + if initialID == "" || continuation.ParentRunID != initialID { + t.Fatalf("native parent identity lost: %q %#v", initialID, continuation) + } + if got := string(evidence.files[path.Join("corrections", initialID, "native", continuation.RunID, "messages.txt")]); !strings.Contains(got, source) || !strings.Contains(got, `{"value":0}`) || !strings.Contains(got, "Return value one.") { + t.Fatalf("visible rejected exchange history lost: %q", got) + } +} + +func TestNativeCaptureRecordsCacheDetailsWithoutExecution(t *testing.T) { + _, store := filesystemProducerCache(t) + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + operation := numberCacheOperation(store) + if _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil); err != nil { + t.Fatal(err) + } + evidence := &nativeEvidence{} + var group promptkit.ExecutionGroup + operation.Execution = contracts.ExecutionOptions{Capture: evidence, Observer: group.Observe, + OnPrepared: func(context.Context, promptkit.PreparedRun) { t.Fatal("cache hit executed preparation callback") }, + } + operation.Assess = func(context.Context, int) (contracts.CandidateAssessment, error) { + t.Fatal("cache hit reassessed") + return contracts.CandidateAssessment{}, nil + } + value, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) + if err != nil || value != 1 { + t.Fatalf("hit=%d, %v", value, err) + } + if len(group.Snapshot().Members) != 0 || nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/exchange.json") != 0 { + t.Fatal("cache hit fabricated execution/history") + } + var details promptkit.PreparedRun + if err := json.Unmarshal(evidence.files["initial/unexecuted/prepared.json"], &details); err != nil || len(details.Messages) == 0 || details.MessageContentHash == "" { + t.Fatalf("hit prepared details unavailable: %#v %v", details, err) + } + if !strings.Contains(string(evidence.files["initial/unexecuted/availability.json"]), `"cache_hit": true`) { + t.Fatal("cache hit not labeled") + } +} + +func TestNativeCaptureRetainsFailuresWithoutInventingResults(t *testing.T) { + for _, missing := range []bool{false, true} { + evidence := &nativeEvidence{} + primary := errors.New("provider stopped") + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, primary + })) + request := nativeProducerInitial() + if missing { + request.PromptID = "not-present" + } + request, capture := CaptureNativeRequest(request, evidence) + result, err := engine.Run(context.Background(), request) + if err == nil || result != nil { + t.Fatalf("expected result-less native failure: %#v, %v", result, err) + } + if captureErr := capture.Finish(result, err, false, false); captureErr != nil { + t.Fatal(captureErr) + } + if nativeEvidenceCount(evidence, "/result.json") != 0 || nativeEvidenceCount(evidence, "/response.txt") != 0 { + t.Fatal("fabricated result/provider body") + } + if nativeEvidenceCount(evidence, "/prepared.json") != map[bool]int{false: 1, true: 0}[missing] { + t.Fatal("prepared availability wrong") + } + found := false + for name, content := range evidence.files { + if strings.HasSuffix(name, "/availability.json") { + found = true + if !strings.Contains(string(content), `"result_available": false`) || !strings.Contains(string(content), `"error":`) { + t.Fatalf("failure evidence unavailable: %s", content) + } + } + } + if !found { + t.Fatal("missing failure evidence") + } + } +} + +func TestNativeCaptureUndeliveredResponseRemainsUnassessed(t *testing.T) { + evidence := &nativeEvidence{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + finished, assessed, fallbackCalls := 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 1}, + Execution: contracts.ExecutionOptions{Capture: evidence, Observer: func(event promptkit.ExecutionEvent) { + if event.Kind == promptkit.ExecutionFinished { + finished++ + if finished == 2 { + cancel() + } + } + }}, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessed++ + return rejectedProducerAssessment(ctx, value) + }, + } + _, outcome, err := RunProducer(ctx, engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil }) + if !errors.Is(err, context.Canceled) || assessed != 1 || fallbackCalls != 0 || outcome.Disposition != "" { + t.Fatalf("undelivered=%#v %v, assessments=%d fallbacks=%d", outcome, err, assessed, fallbackCalls) + } + if nativeEvidenceCount(evidence, "/result.json") != 2 { + t.Fatal("undelivered native response lost") + } + found := false + for name, content := range evidence.files { + if strings.HasSuffix(name, "/availability.json") && strings.Contains(string(content), `"unassessed": true`) { + found = true + } + } + if !found { + t.Fatal("undelivered response not labeled unassessed") + } +} + +func TestNativeCaptureFailureNeverRetriesOrFallsBack(t *testing.T) { + for _, boundary := range []string{"request.json", "/events/", "/prepared.json", "corrections/"} { + t.Run(boundary, func(t *testing.T) { + captureErr := errors.New("evidence write failed") + evidence := &nativeEvidence{fail: func(name string) error { + if strings.Contains(name, boundary) { + return captureErr + } + return nil + }} + calls, assessed, fallbackCalls := 0, 0, 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 2}, Execution: contracts.ExecutionOptions{Capture: evidence}, + Assess: func(ctx context.Context, value int) (contracts.CandidateAssessment, error) { + assessed++ + return rejectedProducerAssessment(ctx, value) + }, + } + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, producerCorrection, func(context.Context, *int) (int, error) { fallbackCalls++; return -1, nil }) + var recording *contracts.CaptureError + if !errors.Is(err, captureErr) || !errors.As(err, &recording) || calls > 1 || fallbackCalls != 0 || outcome.Disposition != "" { + t.Fatalf("capture failure=%#v %v, calls=%d assessed=%d fallback=%d", outcome, err, calls, assessed, fallbackCalls) + } + if boundary == "request.json" && calls != 0 { + t.Fatal("model ran after pre-execution capture failure") + } + }) + } +} + +func TestNativeCaptureSupportsConcurrentOperationsOnOneGroup(t *testing.T) { + evidence := &nativeEvidence{} + var group promptkit.ExecutionGroup + var prepared atomic.Int64 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + const operations = 12 + failures := make(chan error, operations) + var workers sync.WaitGroup + for index := range operations { + workers.Go(func() { + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, Assess: acceptedProducerAssessment, + Execution: contracts.ExecutionOptions{Observer: group.Observe, Capture: scopeEvidence(evidence, fmt.Sprintf("operations/%d", index)), + OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared.Add(1) }, + }, + } + _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil) + failures <- err + }) + } + workers.Wait() + close(failures) + for err := range failures { + if err != nil { + t.Fatal(err) + } + } + summary := group.Snapshot() + if len(summary.Members) != operations || summary.Attempts != operations || !summary.TerminalCoverageComplete || prepared.Load() != operations || nativeEvidenceCount(evidence, "/result.json") != operations { + t.Fatalf("concurrent evidence lost: summary=%#v prepared=%d results=%d", summary, prepared.Load(), nativeEvidenceCount(evidence, "/result.json")) + } +} + +func TestDebugDomainEvidenceDoesNotHeuristicallyRewriteSelectedContent(t *testing.T) { + content := strings.Repeat("Bearer ordinary-application-text sk-ordinary-content ", 1000) + metadata := map[string]any{"token_count": 42, "api_key_note": "ordinary application annotation"} + envelope := debugContentEnvelope([]byte(content), "text/plain", metadata, nil) + decoded, err := base64.StdEncoding.DecodeString(envelope.ContentBase64) + if err != nil || string(decoded) != content || envelope.Metadata["token_count"] != 42 || envelope.Metadata["api_key_note"] != metadata["api_key_note"] { + t.Fatalf("selected domain evidence was rewritten: metadata=%#v error=%v", envelope.Metadata, err) + } + doc := typedTestDocument() + doc.Units[0].Text, doc.Metadata = content, metadata + if captured := debugSourceDocumentEnvelope(doc); captured.Units[0].Text != content || captured.Metadata["token_count"] != 42 { + t.Fatal("parsed source content or ordinary metadata was rewritten") + } + schema := contracts.ArtifactSchema{JSONSchema: []byte(`{"type":"object"}`)} + serialized := debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: contracts.SerializedArtifact{Schema: schema, Content: []byte(content)}}) + if string(serialized.Schema.JSONSchema) != string(schema.JSONSchema) { + t.Fatal("available artifact schema was omitted") + } +} + +func TestNativeCaptureRetainsCorrectionInputsWhenPreparationFails(t *testing.T) { + evidence := &nativeEvidence{} + calls := 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":0}`}, nil + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 1}, Assess: rejectedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}} + _, _, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, func(context.Context, []string) (promptkit.RunRequest, error) { + return promptkit.RunRequest{PromptID: "missing-correction", Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline("constructed correction input")}}, nil + }, nil) + if err == nil || calls != 1 { + t.Fatalf("missing correction preparation: %v calls=%d", err, calls) + } + found := false + for name, content := range evidence.files { + if strings.HasPrefix(name, "corrections/") && strings.HasSuffix(name, "/request.json") && strings.Contains(string(content), "constructed correction input") { + found = true + } + } + if !found { + t.Fatal("constructed correction input lost before preparation") + } +} + +func TestNativeCaptureFailurePreservesProviderCause(t *testing.T) { + primary, recording := errors.New("provider failed"), errors.New("capture failed") + evidence := &nativeEvidence{fail: func(name string) error { + if strings.HasSuffix(name, "/prepared.json") { + return recording + } + return nil + }} + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, primary + })) + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment, Execution: contracts.ExecutionOptions{Capture: evidence}} + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, func(context.Context, *int) (int, error) { t.Fatal("capture failure invoked fallback"); return 0, nil }) + if !errors.Is(err, primary) || !errors.Is(err, recording) || outcome.Disposition != "" { + t.Fatalf("failure causes lost: %#v %v", outcome, err) + } +} diff --git a/internal/framework/pipeline/operation_capture.go b/internal/framework/pipeline/operation_capture.go new file mode 100644 index 00000000..5a5e5818 --- /dev/null +++ b/internal/framework/pipeline/operation_capture.go @@ -0,0 +1,115 @@ +package pipeline + +import ( + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func captureOperationInputs(execution contracts.ExecutionOptions, target typedValidationTarget, binding ModuleBinding, incoming []contracts.SerializedArtifact) error { + if execution.Capture == nil { + return nil + } + references := make(map[string]any, len(target.references.Slots)) + for name, slot := range target.references.Slots { + items := make([]any, 0, len(slot.Items)) + for _, item := range slot.Items { + items = append(items, map[string]any{"reference": item, "content": debugContentEnvelope(item.Content, item.MediaType, nil, nil)}) + } + references[name] = map[string]any{"declaration": slot.Slot, "items": items} + } + inputs := make([]debugSerializedOutput, len(incoming)) + for index, artifact := range incoming { + inputs[index] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact}) + } + var chunk any + if target.chunk != nil { + chunk = debugSourceChunkEnvelope(*target.chunk) + } + return execution.RecordEvidence("inputs.json", map[string]any{ + "source": debugSourceDocumentEnvelope(target.source), + "source_input": debugContentEnvelope(target.sourceInput.Content, target.sourceInput.MediaType, nil, nil), + "chunk": chunk, "chunks": debugSourceChunkEnvelopes(target.chunks), "references": references, + "incoming_artifacts": inputs, "module": binding.Module, "llm_profile": binding.LLMProfile, "metadata": target.metadata, + }) +} + +// Configured options belong to ConfigKit's secret-aware snapshot, not another +// direct serialization of runtime configuration inside composition evidence. +func captureComposition(resolved ResolvedPipeline) ResolvedPipeline { + resolved = cloneResolvedPipeline(resolved) + resolved.Input, resolved.Chunk, resolved.Output = captureBinding(resolved.Input), captureBinding(resolved.Chunk), captureBinding(resolved.Output) + for stepIndex := range resolved.Steps { + for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes { + lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex] + lane.Extract, lane.Merge, lane.Normalize = captureBinding(lane.Extract), captureBinding(lane.Merge), captureBinding(lane.Normalize) + for index := range lane.Validators { + lane.Validators[index] = captureBinding(lane.Validators[index]) + } + } + } + for index := range resolved.ValidatorChains { + for validatorIndex := range resolved.ValidatorChains[index].Validators { + resolved.ValidatorChains[index].Validators[validatorIndex].Binding = captureBinding(resolved.ValidatorChains[index].Validators[validatorIndex].Binding) + } + } + return resolved +} + +func captureBinding(binding ModuleBinding) ModuleBinding { + binding.Options = nil + for index := range binding.Validators.Validators { + binding.Validators.Validators[index] = captureBinding(binding.Validators.Validators[index]) + } + return binding +} + +func captureOperationResult(execution contracts.ExecutionOptions, artifact contracts.SerializedArtifact, outcome contracts.ProducerOutcome, failure error) error { + if execution.Capture == nil { + return nil + } + var value any + if artifact.Content != nil { + value = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: artifact}) + } + errorText := "" + if failure != nil { + errorText = failure.Error() + } + return execution.RecordEvidence("outcome.json", map[string]any{"outcome": outcome, "artifact": value, "error": errorText}) +} + +func captureCacheObservation(writer contracts.EvidenceWriter, observation contracts.CacheObservation) error { + if writer == nil { + return nil + } + errorText := "" + if observation.Err != nil { + errorText = observation.Err.Error() + } + if err := writer.WriteJSON("cache/"+observation.Decision+".json", map[string]any{ + "decision": observation.Decision, "message_content_hash": observation.MessageContentHash, + "error": errorText, + }); err != nil { + return &contracts.CaptureError{Cause: err} + } + if observation.Artifact != nil { + if err := writer.WriteBytes("cache/"+observation.Decision+"-artifact.json", observation.Artifact); err != nil { + return &contracts.CaptureError{Cause: fmt.Errorf("record cached artifact: %w", err)} + } + } + return nil +} + +func assessmentEvidence(assessment contracts.CandidateAssessment) []any { + findings := make([]any, 0, len(assessment.Validators)) + for _, finding := range assessment.Validators { + errorText := "" + if finding.Failure != nil { + errorText = finding.Failure.Error() + } + findings = append(findings, map[string]any{"validator": finding.ValidatorName, "decision": finding.Decision, + "result": finding.Result, "correction_guidance": finding.Result.CorrectionGuidance, "error": errorText}) + } + return findings +} diff --git a/internal/framework/pipeline/operation_capture_test.go b/internal/framework/pipeline/operation_capture_test.go new file mode 100644 index 00000000..ad7a859f --- /dev/null +++ b/internal/framework/pipeline/operation_capture_test.go @@ -0,0 +1,177 @@ +package pipeline + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/source" + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" +) + +func (*nativeEvidence) Enabled() bool { return true } + +func TestOperationInputsRetainSourceReferencesAndIncomingCodecBytes(t *testing.T) { + evidence := &nativeEvidence{} + text := strings.Repeat("sk-ordinary-source-content ", 500) + reference := strings.Repeat("standalone-reference-content ", 500) + doc := typedTestDocument() + doc.Units[0].Text = text + incoming, err := serializeArtifact(assessmentNotesCodec(t), codecNotes{Items: []string{"incoming-value"}}, false) + if err != nil { + t.Fatal(err) + } + target := typedValidationTarget{source: doc, sourceInput: contracts.NewLLMInputMaterial("transcript", "text/plain", []byte(text), "", "input-no-longer-needed"), + chunk: &source.Chunk{ID: "chunk", Content: []byte("chunk material"), Ref: doc.Units[0].Ref}, + references: contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"context": {Items: []contracts.ReferenceItem{{Content: []byte(reference), BindingSource: "reference-no-longer-needed"}}}}}, + } + binding := Binding("producer") + binding.Options = map[string]any{"secret": "configured-option-must-use-native-inspection"} + if err := captureOperationInputs(contracts.ExecutionOptions{Capture: evidence}, target, binding, []contracts.SerializedArtifact{incoming}); err != nil { + t.Fatal(err) + } + content := string(evidence.files["inputs.json"]) + for _, value := range []string{text, reference, "chunk material", "incoming-value"} { + if !strings.Contains(content, value) { + t.Fatalf("missing self-contained input %q", value) + } + } + if strings.Contains(content, "configured-option-must-use-native-inspection") { + t.Fatal("runtime configuration bypassed secret-aware inspection") + } +} + +func TestRunnerCapturesActualArtifactsAndNoExecutionOnReuse(t *testing.T) { + _, store := filesystemProducerCache(t) + calls := 0 + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + calls++ + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) + profile := typedResolutionProfile() + profile.Artifacts = map[string]ArtifactLaneProfile{"notes": registerRunnerNotes(t, catalog, runnerNotesProducer{name: "native", probe: newRunnerProbe(), native: true})} + prepared := prepareRunnerProfile(t, catalog, profile, engine) + for _, reuse := range []bool{false, true} { + evidence := &nativeEvidence{} + var group promptkit.ExecutionGroup + output, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Cache: store, Debug: evidence, Executions: &group, Workers: 1}) + if err != nil || output.FallbackCount != 0 || calls != 1 { + t.Fatalf("run: %#v %v, calls=%d", output, err, calls) + } + if nativeEvidenceCount(evidence, "/inputs.json") != 4 || nativeEvidenceCount(evidence, "/outcome.json") != 4 { + t.Fatal("missing chunk/extract/merge/normalization boundaries") + } + if nativeEvidenceCount(evidence, "/result.json") != map[bool]int{false: 1, true: 0}[reuse] { + t.Fatal("cache reuse fabricated a native execution") + } + for name, content := range evidence.files { + if !strings.HasSuffix(name, "/outcome.json") || strings.Contains(name, "/chunk/") { + continue + } + var outcome struct { + Artifact struct{ Content debugBinaryEnvelope } + } + if err := json.Unmarshal(content, &outcome); err != nil || !strings.Contains(outcome.Artifact.Content.Text, `"1"`) { + t.Fatalf("missing codec artifact for %s: %s %v", name, content, err) + } + } + if reuse && nativeEvidenceCount(evidence, "/cache/hit-artifact.json") != 1 { + t.Fatal("reused artifact bytes not retained") + } + var snapshot promptkit.ExecutionGroupSummary + if err := json.Unmarshal(evidence.files["executions.json"], &snapshot); err != nil { + t.Fatal(err) + } + if len(snapshot.Members) != map[bool]int{false: 1, true: 0}[reuse] || !snapshot.TerminalCoverageComplete { + t.Fatalf("incorrect current-run totals: %#v", snapshot) + } + if len(evidence.files["composition.json"]) == 0 { + t.Fatal("resolved composition unavailable") + } + } +} + +func TestRunnerCaptureDisclosesFallbackAndFailureWithoutInventingAcceptance(t *testing.T) { + for _, fail := range []bool{false, true} { + catalog := typedResolutionCatalog(t, completeTypedCatalogOptions()) + probe := newRunnerProbe() + primary := errors.New("deterministic operation failed") + producer := runnerNotesProducer{name: "notes", probe: probe} + if fail { + producer.failure = primary + } + lane := registerRunnerNotes(t, catalog, producer) + permissive := contracts.ModePermissive + lane.Extract.Mode, lane.Extract.Validators = &permissive, registerRunnerRejection(t, catalog, probe) + profile := typedResolutionProfile() + profile.Artifacts = map[string]ArtifactLaneProfile{"notes": lane} + evidence := &nativeEvidence{} + output, err := New().Run(context.Background(), RunInput{Prepared: prepareRunnerProfile(t, catalog, profile, nil), RawInput: []byte("input"), Debug: evidence, Workers: 1}) + if fail && !errors.Is(err, primary) || !fail && (err != nil || output.FallbackCount != 1) { + t.Fatalf("result=%#v %v", output, err) + } + found := false + for name, content := range evidence.files { + if !strings.Contains(name, "/extract/") || !strings.HasSuffix(name, "/outcome.json") { + continue + } + found = true + var result struct { + Outcome contracts.ProducerOutcome + Error string + Artifact any + } + if err := json.Unmarshal(content, &result); err != nil { + t.Fatal(err) + } + if fail { + if result.Outcome.Disposition != "" || !strings.Contains(result.Error, primary.Error()) || result.Artifact != nil { + t.Fatalf("fabricated failed artifact: %s", content) + } + } else if result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Artifact == nil { + t.Fatalf("fallback lost qualification: %s", content) + } + } + if !found { + t.Fatal("failed/fallback operation not captured") + } + } +} + +func TestInterpretationCaptureFailureCannotConstructPermissiveFallback(t *testing.T) { + primary := errors.New("interpretation evidence failed") + engine := nativeProducerEngine(t, producerClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil + })) + fallbacks := 0 + operation := contracts.ProducerOperation[int]{Controls: contracts.ProducerControls{Mode: contracts.ModePermissive}, Assess: acceptedProducerAssessment} + _, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), func(context.Context, *promptkit.RunResult) (*int, []string, error) { + return nil, nil, &contracts.CaptureError{Cause: primary} + }, nil, func(context.Context, *int) (int, error) { fallbacks++; return -1, nil }) + if !errors.Is(err, primary) || fallbacks != 0 || outcome.Disposition != "" { + t.Fatalf("capture failure became fallback: %#v %v calls=%d", outcome, err, fallbacks) + } +} + +func TestCompositionCaptureDoesNotReserializeOrMutateConfiguredOptions(t *testing.T) { + binding := Binding("configured-module") + binding.Options = map[string]any{"token": "configuration-secret"} + validator := Binding("configured-validator") + validator.Options = map[string]any{"token": "validator-configuration-secret"} + binding.Validators = ValidatorOverride{Set: true, Validators: []ModuleBinding{validator}} + resolved := ResolvedPipeline{Input: binding, Steps: []ResolvedPipelineStep{{ID: "step", ArtifactLanes: []ResolvedArtifactLane{{ID: "lane", Extract: binding}}}}} + content, err := json.Marshal(captureComposition(resolved)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "configuration-secret") || !strings.Contains(string(content), "configured-module") || !strings.Contains(string(content), "configured-validator") { + t.Fatalf("composition lost identities or bypassed native inspection: %s", content) + } + if resolved.Input.Options["token"] != "configuration-secret" || resolved.Input.Validators.Validators[0].Options["token"] != "validator-configuration-secret" { + t.Fatal("capture mutated configured module options") + } +} diff --git a/internal/framework/pipeline/producer.go b/internal/framework/pipeline/producer.go index c08cedc8..b5f6c187 100644 --- a/internal/framework/pipeline/producer.go +++ b/internal/framework/pipeline/producer.go @@ -44,6 +44,22 @@ func RunProducer[T any]( accepted := false evaluate := func(result *promptkit.RunResult) { candidate, guidance, failure, accepted = nil, nil, nil, false + var assessment contracts.CandidateAssessment + defer func() { + if operation.Execution.Capture == nil || result == nil { + return + } + errorText := "" + if failure != nil { + errorText = failure.Error() + } + if err := operation.Execution.RecordEvidence("interpretations/"+result.RunID+".json", map[string]any{ + "run_id": result.RunID, "usable_candidate": candidate != nil, "accepted": accepted, + "correction_guidance": guidance, "assessment": assessmentEvidence(assessment), "error": errorText, + }); err != nil { + assessmentError = errors.Join(assessmentError, err) + } + }() if canceled := ctx.Err(); canceled != nil { failure = canceled return @@ -58,6 +74,10 @@ func RunProducer[T any]( return } if failure != nil { + var captureErr *contracts.CaptureError + if errors.As(failure, &captureErr) { + assessmentError = failure + } candidate, guidance = nil, nil return } @@ -65,7 +85,8 @@ func RunProducer[T any]( failure = errors.New("producer response could not be interpreted as a usable candidate") return } - assessment, err := operation.Assess(ctx, *candidate) + var err error + assessment, err = operation.Assess(ctx, *candidate) if err != nil { assessmentError = err return @@ -85,10 +106,31 @@ func RunProducer[T any]( request := producerRequest(initial, operation.Execution, true) request.CaptureExchange = request.CaptureExchange || (correct != nil && operation.Controls.MaxAdditionalTurns > 0) + request, capture := CaptureNativeRequest(request, scopeEvidence(operation.Execution.Capture, "initial")) + if captureErr := capture.Err(); captureErr != nil { + return zero, contracts.ProducerOutcome{}, captureErr + } var result *promptkit.RunResult var err error var cacheKey string cache := operation.Cache + if captureErr := operation.Execution.RecordEvidence("cache/selection.json", map[string]any{"mode": operation.Controls.Cache, "reuse_eligible": operation.ReuseEligible, "store_available": cache != nil && cache.Store != nil}); captureErr != nil { + return zero, contracts.ProducerOutcome{}, captureErr + } + if cache != nil && operation.Execution.Capture != nil { + owned := *cache + existing := owned.Observe + owned.Observe = func(ctx context.Context, observation contracts.CacheObservation) error { + if err := captureCacheObservation(operation.Execution.Capture, observation); err != nil { + return err + } + if existing != nil { + return existing(ctx, observation) + } + return nil + } + cache = &owned + } if cache != nil && cache.Store != nil && operation.ReuseEligible && operation.Controls.Cache != contracts.CacheBypass { if cache.Encode == nil || cache.Decode == nil { return zero, contracts.ProducerOutcome{}, errors.New("producer cache codec must be supplied") @@ -101,15 +143,21 @@ func RunProducer[T any]( if err == nil { defer prepared.Discard() details := prepared.Details() + if capture != nil { + capture.RetainPrepared(copyPreparedEvidence(details)) + } cacheKey = details.MessageContentHash if captureErr := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "prepared", MessageContentHash: cacheKey, Prepared: &details}); captureErr != nil { - return zero, contracts.ProducerOutcome{}, captureErr + return zero, contracts.ProducerOutcome{}, captureFailure(captureErr, capture.Finish(nil, captureErr, false, false)) } reused, lookupErr := lookupProducerCache(ctx, cache, operation.Controls.Cache, cacheKey) if lookupErr != nil { - return zero, contracts.ProducerOutcome{}, lookupErr + return zero, contracts.ProducerOutcome{}, captureFailure(lookupErr, capture.Finish(nil, lookupErr, false, false)) } if reused != nil { + if captureErr := capture.Finish(nil, nil, false, true); captureErr != nil { + return zero, contracts.ProducerOutcome{}, captureErr + } return *reused, contracts.ProducerOutcome{Disposition: contracts.ProducerAccepted, ReuseEligible: true}, nil } result, err = engine.RunPrepared(ctx, prepared) @@ -121,11 +169,14 @@ func RunProducer[T any]( decision = "ineligible" } if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: decision}); err != nil { - return zero, contracts.ProducerOutcome{}, err + return zero, contracts.ProducerOutcome{}, captureFailure(err, capture.Finish(nil, err, false, false)) } } result, err = engine.Run(ctx, request) } + if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil { + return zero, contracts.ProducerOutcome{}, captureFailure(errors.Join(ctx.Err(), err), captureErr) + } if canceled := ctx.Err(); canceled != nil { return zero, contracts.ProducerOutcome{}, canceled } @@ -142,11 +193,18 @@ func RunProducer[T any]( } if !accepted && err == nil && len(guidance) > 0 && correct != nil && operation.Controls.MaxAdditionalTurns > 0 { + var pendingCapture *NativeCapture + var recordingErr error coordination, coordinateErr := engine.Coordinate(ctx, promptkit.CoordinateRequest{ Start: result.Exchange, AdditionalTurns: operation.Controls.MaxAdditionalTurns, Observer: operation.Execution.Observer, Next: func(_ context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) { // The first callback has no new result. Its findings already exist. if turn.Result != nil { + recordingErr = pendingCapture.Finish(turn.Result, nil, false, false) + pendingCapture = nil + if recordingErr != nil { + return nil, recordingErr + } evaluate(turn.Result) } if assessmentError != nil { @@ -158,8 +216,19 @@ func RunProducer[T any]( // Once another execution is selected, an earlier candidate is not a // terminal fallback source if construction or native execution fails. candidate = nil + correctionPath := "" + if operation.Execution.Capture != nil { + correctionPath = "corrections/" + turn.Exchange.Details().RunID + recordingErr = operation.Execution.RecordEvidence(correctionPath+"/assignment.json", map[string]any{"guidance": guidance}) + if recordingErr != nil { + return nil, recordingErr + } + } next, buildErr := correct(ctx, guidance) if buildErr != nil { + if correctionPath != "" { + recordingErr = operation.Execution.RecordEvidence(correctionPath+"/construction-failure.json", map[string]any{"error": buildErr.Error()}) + } return nil, buildErr } if initial.ProfileID != "" { @@ -169,9 +238,21 @@ func RunProducer[T any]( next.SessionID = initial.SessionID } next = producerRequest(next, operation.Execution, false) + if operation.Execution.Capture != nil { + next, pendingCapture = CaptureNativeRequest(next, scopeEvidence(operation.Execution.Capture, correctionPath)) + if recordingErr = pendingCapture.Err(); recordingErr != nil { + return nil, recordingErr + } + } return &next, nil }, }) + if pendingCapture != nil { + recordingErr = errors.Join(recordingErr, pendingCapture.Finish(coordination.Undelivered, coordinateErr, coordination.Undelivered != nil, false)) + } + if recordingErr != nil { + return zero, contracts.ProducerOutcome{}, captureFailure(coordinateErr, recordingErr) + } if canceled := ctx.Err(); canceled != nil { return zero, contracts.ProducerOutcome{}, canceled } diff --git a/internal/framework/pipeline/producer_cache.go b/internal/framework/pipeline/producer_cache.go index 95043ab0..961cf2bc 100644 --- a/internal/framework/pipeline/producer_cache.go +++ b/internal/framework/pipeline/producer_cache.go @@ -38,9 +38,9 @@ func lookupProducerCache[T any](ctx context.Context, cache *contracts.ProducerCa return nil, canceled } if err != nil { - return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err}) + return nil, observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "unusable", MessageContentHash: hash, Err: err, Artifact: content}) } - if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash}); err != nil { + if err := observeProducerCache(ctx, cache, contracts.CacheObservation{Decision: "hit", MessageContentHash: hash, Artifact: content}); err != nil { return nil, err } return &value, nil @@ -59,7 +59,7 @@ func storeProducerCache[T any](ctx context.Context, cache *contracts.ProducerCac if err := ctx.Err(); err != nil { return err } - observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash} + observation := contracts.CacheObservation{Decision: "stored", MessageContentHash: hash, Artifact: content} if err := cache.Store.Store(ctx, mode, hash, content); err != nil { observation.Decision, observation.Err = "store_failed", err } diff --git a/internal/framework/pipeline/producer_outcome.go b/internal/framework/pipeline/producer_outcome.go index 94a6e660..74ebd6db 100644 --- a/internal/framework/pipeline/producer_outcome.go +++ b/internal/framework/pipeline/producer_outcome.go @@ -21,6 +21,10 @@ func FinishProducerFailure[T any](ctx context.Context, mode contracts.ExecutionM if cause == nil { cause = errors.New("producer did not yield an accepted result") } + var captureErr *contracts.CaptureError + if errors.As(cause, &captureErr) { + return zero, contracts.ProducerOutcome{}, cause + } switch mode { case contracts.ModeStrict: return zero, contracts.ProducerOutcome{}, cause diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 8dfbc580..2c548ca1 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "mime" "path" @@ -14,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "go.maximumdirect.net/promptkit" ) type Registries struct { @@ -44,15 +46,18 @@ type RunInput struct { // on a hit. Shared observers must synchronize concurrent calls. ObserveCache func(context.Context, contracts.DiagnosticOrigin, contracts.CacheObservation) error Execution contracts.ExecutionOptions - Prepared *PreparedPipeline - SourceID string - Path string - RawInput []byte - SessionID string - RunID string - StartedAt time.Time - Metadata map[string]any - Diagnostics []contracts.ProducerDiagnostic + // Executions is the invocation's native group. The runner enrolls its + // requests; callers may use Snapshot without maintaining copied totals. + Executions *promptkit.ExecutionGroup + Prepared *PreparedPipeline + SourceID string + Path string + RawInput []byte + SessionID string + RunID string + StartedAt time.Time + Metadata map[string]any + Diagnostics []contracts.ProducerDiagnostic Debug DebugRecorder // Workers bounds ready extraction and lane-completion jobs together. @@ -107,12 +112,24 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err } debugRecorder = synchronizedDebugRecorder(debugRecorder) input.Debug = debugRecorder + if input.Executions == nil { + input.Executions = &promptkit.ExecutionGroup{} + } + input.Execution.Observer = promptkit.ComposeObservers(input.Execution.Observer, input.Executions.Observe) + defer func() { + if debugRecorder.Enabled() { + if captureErr := debugRecorder.WriteJSON("executions.json", input.Executions.Snapshot()); captureErr != nil { + err = errors.Join(err, &contracts.CaptureError{Cause: captureErr}) + output = failOutput(output) + } + } + }() inputDiagnostics, diagnosticErr := promoteProducerDiagnostics(input.Diagnostics, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageReferences}, nil) if diagnosticErr != nil { return failOutput(output), fmt.Errorf("promote input diagnostics: %w", diagnosticErr) } appendDiagnosticGroups(&output, inputDiagnostics) - if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{ + if err := writeDebugTimed(debugRecorder, "pipeline.json", debugTimedEnvelope{ Stage: "run", StartedAt: startedTime(input.StartedAt), Payload: map[string]any{ @@ -123,6 +140,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err }); err != nil { return failOutput(output), fmt.Errorf("write debug run artifact: %w", err) } + if debugRecorder.Enabled() { + if err := debugRecorder.WriteJSON("composition.json", captureComposition(input.pipeline)); err != nil { + return failOutput(output), &contracts.CaptureError{Cause: err} + } + } adapter := input.Prepared.input var doc *source.SourceDocument @@ -135,8 +157,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err SourceID: input.SourceID, Path: input.Path, Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil), - Options: redactSensitiveMap(input.pipeline.Input.Options), - Metadata: redactSensitiveMap(input.Metadata), + Metadata: input.Metadata, }, }); err != nil { return failOutput(output), fmt.Errorf("write source debug artifact: %w", err) @@ -184,8 +205,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err "cache_mode": input.pipeline.ChunkControls.Cache, "source": debugSourceDocumentEnvelope(doc), "source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil), - "options": redactSensitiveMap(input.pipeline.Chunk.Options), - "metadata": redactSensitiveMap(input.Metadata), + "metadata": input.Metadata, }, }); err != nil { return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err) @@ -282,8 +302,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err } outputDebugPayload := map[string]any{ "normalize_outputs": debugSerializedOutputEnvelopes(output.normalizeOutputs), - "options": redactSensitiveMap(input.pipeline.Output.Options), - "metadata": redactSensitiveMap(input.Metadata), + "metadata": input.Metadata, } if acceptedChunkMap != nil { outputDebugPayload["chunk_map"] = debugSerializedOutputEnvelope(contracts.SerializedOutput{Artifact: *acceptedChunkMap}) diff --git a/internal/framework/pipeline/runner_chunk_plan.go b/internal/framework/pipeline/runner_chunk_plan.go index b7154866..27e028a6 100644 --- a/internal/framework/pipeline/runner_chunk_plan.go +++ b/internal/framework/pipeline/runner_chunk_plan.go @@ -21,15 +21,32 @@ type chunkPlanExecution struct { // Chunk planning remains serial before lane workers. Only a native chunk // producer uses the accepted-artifact cache; deterministic plans run directly. -func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) { +func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (result chunkPlanExecution, err error) { chunker := input.Prepared.chunker - result := chunkPlanExecution{} origin := contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk, ModuleKey: chunker.Key()} - target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: input.Execution} + execution := operationExecution(input, origin, nil) + defer func() { + errorText := "" + if err != nil { + errorText = err.Error() + } + err = captureFailure(err, execution.RecordEvidence("outcome.json", map[string]any{"outcome": result.outcome, "plan": result.plan, "chunks": debugSourceChunkEnvelopes(result.chunks), "error": errorText})) + }() + target := typedValidationTarget{stage: StageChunk, moduleKey: chunker.Key(), source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: input.pipeline.ChunkReferences.ReferenceSet, metadata: input.Metadata, execution: execution} + if err := captureOperationInputs(execution, target, input.pipeline.Chunk, nil); err != nil { + return result, err + } assessChunks := chunkCandidateAssessor(target, input.Prepared.chunkValidators) var assessment contracts.CandidateAssessment - operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible, + candidateNumber := 0 + operation := contracts.ProducerOperation[source.ChunkPlan]{Controls: input.pipeline.ChunkControls, Execution: execution, ReuseEligible: !input.reuseIneligible, Assess: func(ctx context.Context, plan source.ChunkPlan) (contracts.CandidateAssessment, error) { + if execution.Capture != nil { + candidateNumber++ + if err := execution.RecordEvidence(fmt.Sprintf("plans/candidate-%d.json", candidateNumber), plan); err != nil { + return contracts.CandidateAssessment{}, err + } + } _, chunks, err := validateAndMaterializeChunkPlan(doc, plan) if err != nil { if input.pipeline.ChunkExecutionClass == contracts.ExecutionClassDeterministic { diff --git a/internal/framework/pipeline/runner_concurrent.go b/internal/framework/pipeline/runner_concurrent.go index 9e12b9d4..c94b7e58 100644 --- a/internal/framework/pipeline/runner_concurrent.go +++ b/internal/framework/pipeline/runner_concurrent.go @@ -264,10 +264,10 @@ func mergeCompletedLanes(output *RunOutput, completedOutputs []RunOutput) { } } -func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) extractJobResult { +func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, job extractJob) (result extractJobResult) { state := job.lane lane, typed := state.prepared.resolved, state.prepared.typed - result := extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index} + result = extractJobResult{laneIndex: state.index, chunkIndex: job.chunk.Index} if err := ctx.Err(); err != nil { result.err = err return result @@ -283,10 +283,19 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source. return result } references := operationReferenceSet(input, lane.ExtractReferences) - target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution, chunk: &chunk, ref: chunk.Ref} + execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk) + var capturedArtifact contracts.SerializedArtifact + defer func() { + result.err = captureFailure(result.err, captureOperationResult(execution, capturedArtifact, result.outcome, result.err)) + }() + target := typedValidationTarget{stage: StageExtract, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Extract.Module, source: doc, sourceID: doc.ID, sourceInput: chunkInputMaterial(sourceInput, chunk), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution, chunk: &chunk, ref: chunk.Ref} + if err := captureOperationInputs(execution, target, lane.Extract, nil); err != nil { + result.err = err + return result + } var assessment contracts.CandidateAssessment assess := typedCandidateAssessor[any](typed.codec, target, state.prepared.extractValidators) - operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: input.Execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), + operation := contracts.ProducerOperation[any]{Controls: lane.ExtractControls, Execution: execution, ReuseEligible: !input.reuseIneligible && referenceTargetReuseEligible(input, lane.ExtractReferences), Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) { var err error assessment, err = assess(ctx, value) @@ -310,6 +319,7 @@ func (r *Runner) runExtractJob(ctx context.Context, input RunInput, doc *source. result.err = fmt.Errorf("encode extract lane %q chunk %q: %w", lane.ID, chunk.ID, err) return result } + capturedArtifact, result.outcome = stored, outcome result.diagnostics, err = operationDiagnosticGroups(extracted.Diagnostics, cacheDiagnostics, assessment, outcome, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module}, &chunk) if err != nil { result.err = err @@ -348,7 +358,7 @@ func (r *Runner) continueLane(ctx context.Context, input RunInput, doc *source.S reuseEligible: state.reuseEligible, } local.diagnosticGroups = append(local.diagnosticGroups, contracts.CloneDiagnosticCollection(contracts.DiagnosticCollection{Groups: results.diagnostics}).Groups...) - if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "options": redactSensitiveMap(lane.Extract.Options), "metadata": redactSensitiveMap(input.Metadata)}}); err != nil { + if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "input.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"source": debugSourceDocumentEnvelope(doc), "chunks": debugSourceChunkEnvelopes(chunks), "metadata": input.Metadata}}); err != nil { return local, &laneRunError{stage: StageExtract, err: err} } if err := writeDebugTimed(input.Debug, path.Join("extract", fileio.EncodePathComponent(lane.ID), "output.json"), debugTimedEnvelope{Stage: string(StageExtract), StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Extract.Module, StartedAt: time.Now().UTC(), Payload: map[string]any{"outputs": debugExtractArtifacts(results.accepted)}}); err != nil { diff --git a/internal/framework/pipeline/runner_typed.go b/internal/framework/pipeline/runner_typed.go index 029d4b78..a19ada53 100644 --- a/internal/framework/pipeline/runner_typed.go +++ b/internal/framework/pipeline/runner_typed.go @@ -74,8 +74,7 @@ func (r *Runner) continueTypedLane(ctx context.Context, input RunInput, doc *sou return nil } -func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (mergeStageResult, error) { - state := mergeStageResult{} +func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, extracts finalizedExtractResults, output *RunOutput) (state mergeStageResult, err error) { lane, typed := prepared.resolved, prepared.typed references := operationReferenceSet(input, lane.MergeReferences) state.reuseEligible = extracts.reuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.MergeReferences) @@ -90,10 +89,22 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source. if err != nil { return state, err } - target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution} + execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageMerge, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Merge.Module}, nil) + var outcome contracts.ProducerOutcome + defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }() + target := typedValidationTarget{stage: StageMerge, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Merge.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution} + if execution.Capture != nil { + incoming := make([]contracts.SerializedArtifact, len(extracts.accepted)) + for index, value := range extracts.accepted { + incoming[index] = value.artifact + } + if err := captureOperationInputs(execution, target, lane.Merge, incoming); err != nil { + return state, err + } + } var assessment contracts.CandidateAssessment assess := typedCandidateAssessor[any](typed.codec, target, prepared.mergeValidators) - operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible, + operation := contracts.ProducerOperation[any]{Controls: lane.MergeControls, Execution: execution, ReuseEligible: state.reuseEligible, Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) { var err error assessment, err = assess(ctx, value) @@ -130,8 +141,7 @@ func (r *Runner) runMergeStage(ctx context.Context, input RunInput, doc *source. return state, nil } -func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (normalizeStageResult, error) { - state := normalizeStageResult{} +func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, prepared preparedLaneExecutor, merged erasedMergeArtifact, upstreamReuseEligible bool, output *RunOutput) (state normalizeStageResult, err error) { lane, typed := prepared.resolved, prepared.typed references := operationReferenceSet(input, lane.NormalizeReferences) state.reuseEligible = upstreamReuseEligible && !input.reuseIneligible && referenceTargetReuseEligible(input, lane.NormalizeReferences) @@ -142,10 +152,22 @@ func (r *Runner) runNormalizeStage(ctx context.Context, input RunInput, doc *sou if err != nil { return state, err } - target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: input.Execution} + execution := operationExecution(input, contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageNormalize, StepID: input.stepID, LaneID: lane.ID, ModuleKey: lane.Normalize.Module}, nil) + var outcome contracts.ProducerOutcome + defer func() { err = captureFailure(err, captureOperationResult(execution, state.serialized, outcome, err)) }() + target := typedValidationTarget{stage: StageNormalize, stepID: input.stepID, laneID: lane.ID, moduleKey: lane.Normalize.Module, source: doc, sourceID: doc.ID, sourceInput: sourceInput.Clone(), sessionID: sessionID, references: references, metadata: input.Metadata, execution: execution} + if execution.Capture != nil { + incoming, err := serializeArtifact(typed.codec, merged.Value, false) + if err != nil { + return state, fmt.Errorf("encode normalization input: %w", err) + } + if err := captureOperationInputs(execution, target, lane.Normalize, []contracts.SerializedArtifact{incoming}); err != nil { + return state, err + } + } var assessment contracts.CandidateAssessment assess := typedCandidateAssessor[any](typed.codec, target, prepared.normalizeValidators) - operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: input.Execution, ReuseEligible: state.reuseEligible, + operation := contracts.ProducerOperation[any]{Controls: lane.NormalizeControls, Execution: execution, ReuseEligible: state.reuseEligible, Assess: func(ctx context.Context, value any) (contracts.CandidateAssessment, error) { var err error assessment, err = assess(ctx, value) diff --git a/internal/modules/dnd/normalize/itemregistry/native_execution_test.go b/internal/modules/dnd/normalize/itemregistry/native_execution_test.go index 162d0171..51637d56 100644 --- a/internal/modules/dnd/normalize/itemregistry/native_execution_test.go +++ b/internal/modules/dnd/normalize/itemregistry/native_execution_test.go @@ -2,13 +2,18 @@ package itemregistry import ( "context" + "encoding/json" "errors" + "os" + "path/filepath" "reflect" "strings" "testing" + "time" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" itemcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/itemregistry" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/items/identity" @@ -39,6 +44,43 @@ func itemNames(value dnd.ItemRegistry) []string { return names } +func TestNormalizationCaptureRetainsFixedMappingAndEveryProposal(t *testing.T) { + recorder, err := frameworkdebug.Start(t.TempDir(), "normalization", time.Unix(1, 0)) + if err != nil { + t.Fatal(err) + } + client := &recordingNormalizerClient{responses: []string{invalidItemProposal, safeFirstProposal}} + request := normalizeRequestWithSource(mergedItems(), semanticDocument()) + request.Operation.Execution.Capture = recorder + request.Operation.Controls.MaxAdditionalTurns = 1 + result, err := newNormalizer(t, client).Normalize(context.Background(), request) + if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted { + t.Fatalf("normalization: %#v %v", result, err) + } + content, err := os.ReadFile(filepath.Join(recorder.Path(), "normalization", "originals.json")) + if err != nil { + t.Fatal(err) + } + var originals dnd.ItemRegistry + if err := json.Unmarshal(content, &originals); err != nil || len(originals.Items) != 3 { + t.Fatalf("fixed originals missing: %#v %v", originals, err) + } + content, err = os.ReadFile(filepath.Join(recorder.Path(), "normalization", "preparation.json")) + if err != nil { + t.Fatal(err) + } + var preparation struct { + Mappings []any `json:"candidate_mappings"` + } + if err := json.Unmarshal(content, &preparation); err != nil || len(preparation.Mappings) != 3 { + t.Fatalf("fixed handle mapping missing: %s %v", content, err) + } + proposals, err := os.ReadDir(filepath.Join(recorder.Path(), "normalization", "proposals")) + if err != nil || len(proposals) != 2 { + t.Fatalf("revised proposals lost: %v %v", proposals, err) + } +} + func TestNativeNormalizationReassessesRevisionsAndUsesOnlyTheFinalProposal(t *testing.T) { finalPartial := `{"duplicate_groups":[{"candidate_numbers":[2,3],"canonical_candidate_number":3},{"candidate_numbers":[1,99],"canonical_candidate_number":1}]}` for _, test := range []struct { diff --git a/internal/modules/dnd/normalize/itemregistry/normalizer.go b/internal/modules/dnd/normalize/itemregistry/normalizer.go index 669346c7..c51dc085 100644 --- a/internal/modules/dnd/normalize/itemregistry/normalizer.go +++ b/internal/modules/dnd/normalize/itemregistry/normalizer.go @@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize order := shared.NewSourceRefOrder(req.Source) records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) + if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err + } findings := append([]diagnostics.Finding(nil), baseFindings...) var advisories []diagnostics.Finding discardedGroups := -1 @@ -134,6 +137,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err != nil { return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } + if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{ + "disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(), + "candidate_mappings": preparation.CandidateMappings(), "candidates": candidates, + }); err != nil { + return contracts.TypedNormalizeResult[dnd.ItemRegistry]{}, err + } switch preparation.Disposition() { case semanticreconcile.InsufficientCandidates: return noModel() @@ -152,6 +161,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return nil, nil, err } assessment := preparation.Assess(response) + if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil { + return nil, nil, err + } applied, semanticFindings, advisoryFindings, rejectedGroups, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) if err != nil { return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) diff --git a/internal/modules/dnd/normalize/locationregistry/normalizer.go b/internal/modules/dnd/normalize/locationregistry/normalizer.go index a4fb989c..ee121b92 100644 --- a/internal/modules/dnd/normalize/locationregistry/normalizer.go +++ b/internal/modules/dnd/normalize/locationregistry/normalizer.go @@ -78,6 +78,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize order := shared.NewSourceRefOrder(req.Source) records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) + if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err + } findings := append([]diagnostics.Finding(nil), baseFindings...) discardedGroups := -1 var omissionDetails []string @@ -133,6 +136,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err != nil { return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } + if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{ + "disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(), + "candidate_mappings": preparation.CandidateMappings(), "candidates": candidates, + }); err != nil { + return contracts.TypedNormalizeResult[dnd.LocationRegistry]{}, err + } switch preparation.Disposition() { case semanticreconcile.InsufficientCandidates: return noModel() @@ -151,6 +160,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return nil, nil, err } assessment := preparation.Assess(response) + if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil { + return nil, nil, err + } applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) if err != nil { return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) diff --git a/internal/modules/dnd/normalize/npcregistry/normalizer.go b/internal/modules/dnd/normalize/npcregistry/normalizer.go index e08bce64..3e61d112 100644 --- a/internal/modules/dnd/normalize/npcregistry/normalizer.go +++ b/internal/modules/dnd/normalize/npcregistry/normalizer.go @@ -77,6 +77,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize order := shared.NewSourceRefOrder(req.Source) records, baseFindings := preprocessRecords(merged, order) deterministic := recordList(records) + if err := req.Operation.Execution.RecordEvidence("normalization/originals.json", deterministic); err != nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err + } findings := append([]diagnostics.Finding(nil), baseFindings...) discardedGroups := -1 var omissionDetails []string @@ -132,6 +135,12 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize if err != nil { return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, normalizerErrorf("prepare semantic reconciliation: %w", err) } + if err := req.Operation.Execution.RecordEvidence("normalization/preparation.json", map[string]any{ + "disposition": preparation.Disposition(), "candidate_number_range": preparation.CandidateNumberRange(), + "candidate_mappings": preparation.CandidateMappings(), "candidates": candidates, + }); err != nil { + return contracts.TypedNormalizeResult[dnd.NPCRegistry]{}, err + } switch preparation.Disposition() { case semanticreconcile.InsufficientCandidates: return noModel() @@ -150,6 +159,9 @@ func (n *Normalizer) Normalize(ctx context.Context, req contracts.TypedNormalize return nil, nil, err } assessment := preparation.Assess(response) + if err := req.Operation.Execution.RecordEvidence("normalization/proposals/"+result.RunID+".json", map[string]any{"proposal": response, "issues": assessment.Issues(), "discarded_group_count": assessment.DiscardedGroupCount()}); err != nil { + return nil, nil, err + } applied, semanticFindings, err := applyReconciliationPlan(assessment.Plan(), records, envelopes, order) if err != nil { return nil, nil, normalizerErrorf("apply semantic reconciliation plan: %w", err) diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go index e8c18688..5848f05e 100644 --- a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/validator.go @@ -3,6 +3,7 @@ package combat_semantics import ( "context" "encoding/json" + "errors" "strings" "go.maximumdirect.net/promptkit" @@ -50,12 +51,19 @@ func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationR } material := shared.TranscriptPromptMaterial(sourceInput) - result, err := v.engine.Run(ctx, promptkit.RunRequest{ + request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{ PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))}, Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries, Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared, - }) + }, req.Execution.Capture) + if err := capture.Err(); err != nil { + return contracts.ValidationResult{}, err + } + result, err := v.engine.Run(ctx, request) + if captureErr := capture.Finish(result, err, ctx.Err() != nil, false); captureErr != nil { + return contracts.ValidationResult{}, errors.Join(ctx.Err(), err, captureErr) + } if canceled := ctx.Err(); canceled != nil { return contracts.ValidationResult{}, canceled }