Capture native execution facts and complete scoped debug evidence

This commit is contained in:
2026-11-22 03:19:43 +00:00
parent 8680060400
commit 9f66421f4f
24 changed files with 1688 additions and 543 deletions

View File

@@ -243,9 +243,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
debugPath := capture.Path()
var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder()
if capture.Enabled() {
debugRecorder = capture
debugRecorder = newRunEvidenceRecorder(capture, loaded.Result())
}
if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil {
version, err := buildinfo.Version()
if err != nil {
return fail(err)
}
if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt, "selected_lanes": only, "llm_profile": llmProfile.value, "llm_session_id": requestedSessionID.value, "execution_overrides": runtimeOverrides}); err != nil {
return fail(err)
}
if err := captureRunConfiguration(capture, loaded.Result()); err != nil {
return fail(err)
}
@@ -299,7 +306,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
if err != nil {
return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
}
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
if err != nil {
return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))

157
internal/cli/run_capture.go Normal file
View File

@@ -0,0 +1,157 @@
package cli
import (
"bytes"
"fmt"
"maps"
"slices"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
// captureRunConfiguration inspects the original, frozen application load. It
// neither reloads resources nor serializes typed settings containing credentials.
// Shared loaders remain independent of run storage.
func captureRunConfiguration(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) error {
if !recorder.Enabled() {
return nil
}
if result == nil {
return fmt.Errorf("capture configuration: application result is unavailable")
}
var application bytes.Buffer
if err := result.WriteEffective(&application); err != nil {
return fmt.Errorf("inspect application configuration: %w", err)
}
if err := recorder.WriteBytes("configuration/application.yml", application.Bytes()); err != nil {
return err
}
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
if !ok {
return fmt.Errorf("capture configuration: PromptKit inspection is unavailable")
}
var prompts bytes.Buffer
if err := view.WriteEffective(&prompts); err != nil {
return fmt.Errorf("inspect PromptKit configuration: %w", err)
}
if err := recorder.WriteBytes("configuration/promptkit.yml", prompts.Bytes()); err != nil {
return err
}
return recorder.WriteJSON("configuration/sources.json", result.Sources())
}
type runEvidenceRecorder struct {
*frameworkdebug.Recorder
profiles map[string][][]string
backends map[string][][]string
}
// Keep ConfigKit-marked provider settings omitted in native execution evidence,
// too. These are exact configuration paths from the adopted definition, not
// searches for credential-looking strings in application content.
func newRunEvidenceRecorder(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) pipeline.DebugRecorder {
writer := &runEvidenceRecorder{Recorder: recorder, profiles: map[string][][]string{}, backends: map[string][][]string{}}
if result == nil {
return writer
}
sensitive := config.Definition("").Sensitive
paths := result.Metadata().HistoricalPaths()
register := func(masks map[string][][]string, id string, prefix configkit.Path) {
for _, field := range paths {
if !strings.HasPrefix(field.String(), prefix.String()+"/") || !sensitive(field) {
continue
}
segments := strings.Split(strings.TrimPrefix(field.String(), prefix.String()+"/"), "/")
for index, segment := range segments {
segments[index] = strings.ReplaceAll(strings.ReplaceAll(segment, "~1", "/"), "~0", "~")
}
masks[id] = append(masks[id], segments)
}
}
root := configkit.Keys("promptkit")
for index, profile := range result.Config.PromptKit.Profiles {
register(writer.profiles, profile.ID, root.Key("profiles").Index(index).Key("extra_params"))
}
for index, backend := range result.Config.PromptKit.Backends {
register(writer.backends, backend.ID, root.Key("backends").Index(index).Key("extra_params"))
}
for namespace, collection := range result.Config.PromptKit.Collections {
prefix := root.Key("collections").Key(namespace)
for index, profile := range collection.Profiles {
register(writer.profiles, namespace+"/"+profile.ID, prefix.Key("profiles").Index(index).Key("extra_params"))
}
for index, backend := range collection.Backends {
register(writer.backends, namespace+"/"+backend.ID, prefix.Key("backends").Index(index).Key("extra_params"))
}
}
return writer
}
func (writer *runEvidenceRecorder) WriteJSON(name string, payload any) error {
switch value := payload.(type) {
case *promptkit.PreparedRun:
copy := *value
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
payload = copy
case *promptkit.RunResult:
copy := *value
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
payload = copy
}
return writer.Recorder.WriteJSON(name, payload)
}
func (writer *runEvidenceRecorder) target(target promptkit.ExecutionTarget, profile, backend string) promptkit.ExecutionTarget {
for _, segments := range writer.backends[backend] {
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
}
for _, segments := range writer.profiles[profile] {
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
}
return target
}
// Copy only changed map branches; never mutate the native snapshot delivered to
// another observer. Ordinary siblings and all application text remain untouched.
func omitEvidenceSetting(values map[string]any, segments []string) map[string]any {
if len(segments) == 0 || values == nil {
return values
}
value, found := values[segments[0]]
if !found {
return values
}
copy := maps.Clone(values)
if len(segments) == 1 {
copy[segments[0]] = "[REDACTED]"
return copy
}
copy[segments[0]] = omitEvidenceValue(value, segments[1:])
return copy
}
func omitEvidenceValue(value any, segments []string) any {
if len(segments) == 0 {
return "[REDACTED]"
}
switch value := value.(type) {
case map[string]any:
return omitEvidenceSetting(value, segments)
case []any:
index, err := strconv.Atoi(segments[0])
if err != nil || index < 0 || index >= len(value) {
return value
}
copy := slices.Clone(value)
copy[index] = omitEvidenceValue(copy[index], segments[1:])
return copy
default:
return value
}
}

View File

@@ -0,0 +1,128 @@
package cli
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6
promptkit:
profiles:
- id: fixture
model: sk-ordinary-model-name
extra_params: {token: captured-secret-value}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
// Mutating public typed settings must not change the frozen inspection.
mutated := "mutated-after-load"
result.Config.PromptKit.Profiles[0].Model = &mutated
recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(recorder, result); err != nil {
t.Fatal(err)
}
for _, name := range []string{"application.yml", "promptkit.yml"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") {
t.Fatalf("configuration capture lost frozen/native secret handling: %s", content)
}
}
if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(nil, nil); err != nil {
t.Fatalf("disabled capture inspected configuration: %v", err)
}
}
type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return client(ctx, request)
}
func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6
promptkit:
collections:
fixture:
profiles:
- id: base
endpoint: https://example.invalid/v1
model: sk-ordinary-model-name
api_key_required: false
extra_params:
token: configured-secret
items: [{authorization: nested-secret, caption: ordinary-caption}]
- id: chosen
extends: {id: base}
prompts:
- id: request
default_profile: fixture/chosen
messages: [{role: user, content: sk-ordinary-application-content}]
output: {format: text, validation_mode: none}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" {
t.Fatal("capture changed native execution")
}
return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil
})))
if err != nil {
t.Fatal(err)
}
recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result))
response, err := engine.Run(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if err := capture.Finish(response, nil, false, false); err != nil {
t.Fatal(err)
}
for _, name := range []string{"prepared.json", "result.json"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name))
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} {
if strings.Contains(string(content), secret) {
t.Fatalf("ConfigKit/native secret leaked in %s", name)
}
}
if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") {
t.Fatalf("ordinary native settings were rewritten: %s", content)
}
}
if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" {
t.Fatal("projection mutated the caller's native snapshot")
}
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt"))
if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") {
t.Fatalf("application content rewritten: %s %v", content, err)
}
}