Files
notarius/internal/cli/run_capture_test.go

177 lines
7.1 KiB
Go
Raw Normal View History

package cli
import (
"context"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestRunCaptureRoutesRootAndCollectionSensitivityMasks(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("mixed resources", []byte(`version: 6
promptkit:
profiles:
- id: chosen
extra_params:
token: root-profile
nested/key:
a~b: [{authorization: root-nested, caption: ordinary-caption}]
backends:
- id: chosen
endpoint: https://example.invalid/v1
extra_params: {password: root-backend}
collections:
fixture:
profiles:
- id: chosen
extra_params: {token: collection-profile}
backends:
- id: chosen
endpoint: https://example.invalid/v1
extra_params: {password: collection-backend}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
writer := newRunEvidenceRecorder(nil, result).(*runEvidenceRecorder)
nested := map[string]any{"a~b": []any{map[string]any{"authorization": "root-nested", "caption": "ordinary-caption"}}}
for _, namespace := range []string{"", "fixture/"} {
target := promptkit.ExecutionTarget{ExtraParams: map[string]any{"token": "original-token", "password": "original-password", "nested/key": nested, "caption": "ordinary-caption"}}
masked := writer.target(target, namespace+"chosen", namespace+"chosen")
if masked.ExtraParams["token"] != "[REDACTED]" || masked.ExtraParams["password"] != "[REDACTED]" || masked.ExtraParams["caption"] != "ordinary-caption" {
t.Fatalf("misrouted resource masks for %q: %#v", namespace, masked)
}
wantNested := nested
if namespace == "" {
wantNested = map[string]any{"a~b": []any{map[string]any{"authorization": "[REDACTED]", "caption": "ordinary-caption"}}}
}
if !reflect.DeepEqual(masked.ExtraParams["nested/key"], wantNested) {
t.Fatalf("nested mask crossed resource boundary for %q: %#v", namespace, masked)
}
if target.ExtraParams["token"] != "original-token" || nested["a~b"].([]any)[0].(map[string]any)["authorization"] != "root-nested" {
t.Fatal("native snapshot was mutated")
}
}
}
func TestRunCaptureUsesFrozenSecretAwareConfiguration(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("captured application", []byte(`version: 6
promptkit:
profiles:
- id: fixture
model: sk-ordinary-model-name
extra_params: {token: captured-secret-value}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
// Mutating public typed settings must not change the frozen inspection.
mutated := "mutated-after-load"
result.Config.PromptKit.Profiles[0].Model = &mutated
recorder, err := frameworkdebug.Start(t.TempDir(), "capture", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(recorder, result); err != nil {
t.Fatal(err)
}
for _, name := range []string{"application.yml", "promptkit.yml"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "configuration", name))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(content), "captured-secret-value") || strings.Contains(string(content), "mutated-after-load") || !strings.Contains(string(content), "sk-ordinary-model-name") {
t.Fatalf("configuration capture lost frozen/native secret handling: %s", content)
}
}
if _, err := os.Stat(filepath.Join(recorder.Path(), "configuration", "sources.json")); err != nil {
t.Fatal(err)
}
if err := captureRunConfiguration(nil, nil); err != nil {
t.Fatalf("disabled capture inspected configuration: %v", err)
}
}
type captureClient func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
func (client captureClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return client(ctx, request)
}
func TestRunCapturePreservesNativeConfigurationSecretMarksInExecutionSnapshots(t *testing.T) {
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("application", []byte(`version: 6
promptkit:
collections:
fixture:
profiles:
- id: base
endpoint: https://example.invalid/v1
model: sk-ordinary-model-name
api_key_required: false
extra_params:
token: configured-secret
items: [{authorization: nested-secret, caption: ordinary-caption}]
- id: chosen
extends: {id: base}
prompts:
- id: request
default_profile: fixture/chosen
messages: [{role: user, content: sk-ordinary-application-content}]
output: {format: text, validation_mode: none}
`))}, configkit.LayeredOptions[config.Document]{Definition: config.Definition(t.TempDir())})
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.ExtraParams["token"] != "configured-secret" || request.APIKey != "direct-secret" {
t.Fatal("capture changed native execution")
}
return &promptkit.GenerateResponse{Content: "token and sk-ordinary-response-content"}, nil
})))
if err != nil {
t.Fatal(err)
}
recorder, err := frameworkdebug.Start(t.TempDir(), "run", time.Unix(1, 0))
if err != nil {
t.Fatal(err)
}
request, capture := pipeline.CaptureNativeRequest(promptkit.RunRequest{PromptID: "fixture/request", APIKey: "direct-secret"}, newRunEvidenceRecorder(recorder, result))
response, err := engine.Run(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if err := capture.Finish(response, nil, false, false); err != nil {
t.Fatal(err)
}
for _, name := range []string{"prepared.json", "result.json"} {
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, name))
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{"configured-secret", "nested-secret", "direct-secret"} {
if strings.Contains(string(content), secret) {
t.Fatalf("ConfigKit/native secret leaked in %s", name)
}
}
if !strings.Contains(string(content), "sk-ordinary-model-name") || !strings.Contains(string(content), "ordinary-caption") {
t.Fatalf("ordinary native settings were rewritten: %s", content)
}
}
if response.EffectiveModelParams.ExtraParams["token"] != "configured-secret" {
t.Fatal("projection mutated the caller's native snapshot")
}
content, err := os.ReadFile(filepath.Join(recorder.Path(), "native", response.RunID, "messages.txt"))
if err != nil || !strings.Contains(string(content), "sk-ordinary-application-content") || !strings.Contains(string(content), "token and sk-ordinary-response-content") {
t.Fatalf("application content rewritten: %s %v", content, err)
}
}