Capture native execution facts and complete scoped debug evidence
This commit is contained in:
@@ -243,9 +243,16 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
debugPath := capture.Path()
|
||||
var debugRecorder pipeline.DebugRecorder = pipeline.NoopDebugRecorder()
|
||||
if capture.Enabled() {
|
||||
debugRecorder = capture
|
||||
debugRecorder = newRunEvidenceRecorder(capture, loaded.Result())
|
||||
}
|
||||
if err := capture.WriteJSON("context.json", map[string]any{"pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt}); err != nil {
|
||||
version, err := buildinfo.Version()
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if err := capture.WriteJSON("context.json", map[string]any{"application_version": version, "pipeline": pipelineID, "input_path": *inputPath, "config_path": loadedConfigPath, "run_id": runID, "started_at": startedAt, "selected_lanes": only, "llm_profile": llmProfile.value, "llm_session_id": requestedSessionID.value, "execution_overrides": runtimeOverrides}); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if err := captureRunConfiguration(capture, loaded.Result()); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
@@ -299,7 +306,6 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
|
||||
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
|
||||
|
||||
157
internal/cli/run_capture.go
Normal file
157
internal/cli/run_capture.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// captureRunConfiguration inspects the original, frozen application load. It
|
||||
// neither reloads resources nor serializes typed settings containing credentials.
|
||||
// Shared loaders remain independent of run storage.
|
||||
func captureRunConfiguration(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) error {
|
||||
if !recorder.Enabled() {
|
||||
return nil
|
||||
}
|
||||
if result == nil {
|
||||
return fmt.Errorf("capture configuration: application result is unavailable")
|
||||
}
|
||||
var application bytes.Buffer
|
||||
if err := result.WriteEffective(&application); err != nil {
|
||||
return fmt.Errorf("inspect application configuration: %w", err)
|
||||
}
|
||||
if err := recorder.WriteBytes("configuration/application.yml", application.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
view, ok := result.InspectSubtree(configkit.Keys("promptkit"))
|
||||
if !ok {
|
||||
return fmt.Errorf("capture configuration: PromptKit inspection is unavailable")
|
||||
}
|
||||
var prompts bytes.Buffer
|
||||
if err := view.WriteEffective(&prompts); err != nil {
|
||||
return fmt.Errorf("inspect PromptKit configuration: %w", err)
|
||||
}
|
||||
if err := recorder.WriteBytes("configuration/promptkit.yml", prompts.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
return recorder.WriteJSON("configuration/sources.json", result.Sources())
|
||||
}
|
||||
|
||||
type runEvidenceRecorder struct {
|
||||
*frameworkdebug.Recorder
|
||||
profiles map[string][][]string
|
||||
backends map[string][][]string
|
||||
}
|
||||
|
||||
// Keep ConfigKit-marked provider settings omitted in native execution evidence,
|
||||
// too. These are exact configuration paths from the adopted definition, not
|
||||
// searches for credential-looking strings in application content.
|
||||
func newRunEvidenceRecorder(recorder *frameworkdebug.Recorder, result *configkit.Result[config.Document]) pipeline.DebugRecorder {
|
||||
writer := &runEvidenceRecorder{Recorder: recorder, profiles: map[string][][]string{}, backends: map[string][][]string{}}
|
||||
if result == nil {
|
||||
return writer
|
||||
}
|
||||
sensitive := config.Definition("").Sensitive
|
||||
paths := result.Metadata().HistoricalPaths()
|
||||
register := func(masks map[string][][]string, id string, prefix configkit.Path) {
|
||||
for _, field := range paths {
|
||||
if !strings.HasPrefix(field.String(), prefix.String()+"/") || !sensitive(field) {
|
||||
continue
|
||||
}
|
||||
segments := strings.Split(strings.TrimPrefix(field.String(), prefix.String()+"/"), "/")
|
||||
for index, segment := range segments {
|
||||
segments[index] = strings.ReplaceAll(strings.ReplaceAll(segment, "~1", "/"), "~0", "~")
|
||||
}
|
||||
masks[id] = append(masks[id], segments)
|
||||
}
|
||||
}
|
||||
root := configkit.Keys("promptkit")
|
||||
for index, profile := range result.Config.PromptKit.Profiles {
|
||||
register(writer.profiles, profile.ID, root.Key("profiles").Index(index).Key("extra_params"))
|
||||
}
|
||||
for index, backend := range result.Config.PromptKit.Backends {
|
||||
register(writer.backends, backend.ID, root.Key("backends").Index(index).Key("extra_params"))
|
||||
}
|
||||
for namespace, collection := range result.Config.PromptKit.Collections {
|
||||
prefix := root.Key("collections").Key(namespace)
|
||||
for index, profile := range collection.Profiles {
|
||||
register(writer.profiles, namespace+"/"+profile.ID, prefix.Key("profiles").Index(index).Key("extra_params"))
|
||||
}
|
||||
for index, backend := range collection.Backends {
|
||||
register(writer.backends, namespace+"/"+backend.ID, prefix.Key("backends").Index(index).Key("extra_params"))
|
||||
}
|
||||
}
|
||||
return writer
|
||||
}
|
||||
|
||||
func (writer *runEvidenceRecorder) WriteJSON(name string, payload any) error {
|
||||
switch value := payload.(type) {
|
||||
case *promptkit.PreparedRun:
|
||||
copy := *value
|
||||
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
|
||||
payload = copy
|
||||
case *promptkit.RunResult:
|
||||
copy := *value
|
||||
copy.EffectiveModelParams = writer.target(copy.EffectiveModelParams, copy.SelectedProfileID, copy.SelectedBackendID)
|
||||
payload = copy
|
||||
}
|
||||
return writer.Recorder.WriteJSON(name, payload)
|
||||
}
|
||||
|
||||
func (writer *runEvidenceRecorder) target(target promptkit.ExecutionTarget, profile, backend string) promptkit.ExecutionTarget {
|
||||
for _, segments := range writer.backends[backend] {
|
||||
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
|
||||
}
|
||||
for _, segments := range writer.profiles[profile] {
|
||||
target.ExtraParams = omitEvidenceSetting(target.ExtraParams, segments)
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
// Copy only changed map branches; never mutate the native snapshot delivered to
|
||||
// another observer. Ordinary siblings and all application text remain untouched.
|
||||
func omitEvidenceSetting(values map[string]any, segments []string) map[string]any {
|
||||
if len(segments) == 0 || values == nil {
|
||||
return values
|
||||
}
|
||||
value, found := values[segments[0]]
|
||||
if !found {
|
||||
return values
|
||||
}
|
||||
copy := maps.Clone(values)
|
||||
if len(segments) == 1 {
|
||||
copy[segments[0]] = "[REDACTED]"
|
||||
return copy
|
||||
}
|
||||
copy[segments[0]] = omitEvidenceValue(value, segments[1:])
|
||||
return copy
|
||||
}
|
||||
|
||||
func omitEvidenceValue(value any, segments []string) any {
|
||||
if len(segments) == 0 {
|
||||
return "[REDACTED]"
|
||||
}
|
||||
switch value := value.(type) {
|
||||
case map[string]any:
|
||||
return omitEvidenceSetting(value, segments)
|
||||
case []any:
|
||||
index, err := strconv.Atoi(segments[0])
|
||||
if err != nil || index < 0 || index >= len(value) {
|
||||
return value
|
||||
}
|
||||
copy := slices.Clone(value)
|
||||
copy[index] = omitEvidenceValue(copy[index], segments[1:])
|
||||
return copy
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
128
internal/cli/run_capture_test.go
Normal file
128
internal/cli/run_capture_test.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) {
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: fixture
|
||||
model: sk-ordinary-model-name
|
||||
extra_params: {token: captured-secret-value}
|
||||
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Mutating public typed settings must not change the frozen inspection.
|
||||
mutated := "mutated-after-load"
|
||||
result.Config.PromptKit.Profiles[0].Model = &mutated
|
||||
recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := captureRunConfiguration(recorder, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"application.yml", "promptkit.yml"} {
|
||||
content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") {
|
||||
t.Fatalf("configuration capture lost frozen/native secret handling: %s", content)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := captureRunConfiguration(nil, nil); err != nil {
|
||||
t.Fatalf("disabled capture inspected configuration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
|
||||
|
||||
func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
return client(ctx, request)
|
||||
}
|
||||
|
||||
func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) {
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6
|
||||
promptkit:
|
||||
collections:
|
||||
fixture:
|
||||
profiles:
|
||||
- id: base
|
||||
endpoint: https://example.invalid/v1
|
||||
model: sk-ordinary-model-name
|
||||
api_key_required: false
|
||||
extra_params:
|
||||
token: configured-secret
|
||||
items: [{authorization: nested-secret, caption: ordinary-caption}]
|
||||
- id: chosen
|
||||
extends: {id: base}
|
||||
prompts:
|
||||
- id: request
|
||||
default_profile: fixture/chosen
|
||||
messages: [{role: user, content: sk-ordinary-application-content}]
|
||||
output: {format: text, validation_mode: none}
|
||||
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" {
|
||||
t.Fatal("capture changed native execution")
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result))
|
||||
response, err := engine.Run(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := capture.Finish(response, nil, false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"prepared.json", "result.json"} {
|
||||
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} {
|
||||
if strings.Contains(string(content), secret) {
|
||||
t.Fatalf("ConfigKit/native secret leaked in %s", name)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") {
|
||||
t.Fatalf("ordinary native settings were rewritten: %s", content)
|
||||
}
|
||||
}
|
||||
if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" {
|
||||
t.Fatal("projection mutated the caller's native snapshot")
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt"))
|
||||
if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") {
|
||||
t.Fatalf("application content rewritten: %s %v", content, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user