package promptkit_test import ( "bytes" "context" "errors" "fmt" "io" "strings" "sync" "sync/atomic" "testing" "testing/fstest" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{ "backends": []any{map[string]any{ "id": "local", "endpoint": " HTTPS://example.invalid/v1 ", "api_key_env": "OPTIONAL_KEY", "extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"}, }}, "collections": map[string]any{"literal": map[string]any{ "backends": []any{ map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}}, map[string]any{"id": "child", "extends": map[string]any{"id": "local"}}, }, "profiles": []any{ map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"}, map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}}, }, }}, })}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{ "collections": map[string]any{"literal": map[string]any{ "backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}}, "profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}}, }}, })}}}, }, }}, promptkit.WithBundledOpenRouter(false)) if err != nil { t.Fatalf("construct layered engine: %v", err) } view, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model") model, found := view.Explain(modelPath) if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 { t.Fatalf("layered model explanation = (%+v, %t)", model, found) } inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model") inherited, found := view.Explain(inheritedPath) if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) { t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found) } if inheritedPath.String() != "/collections/literal/profiles/1/model" { t.Fatalf("literal-key path = %q", inheritedPath) } secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token") endpointPath := configkit.Keys("backends").Index(0).Key("endpoint") endpoint, found := view.Explain(endpointPath) if !found || !strings.Contains(endpoint.Effective.YAML, "https://example.invalid/v1") || strings.Contains(fmt.Sprintf("%+v", view.Metadata().History(endpointPath)), "private-sentinel") { t.Fatalf("normalized endpoint explanation = %+v", endpoint) } secret, found := view.Explain(secretPath) if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") { t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found) } namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token") namespacedSecret, found := view.Explain(namespacedSecretPath) if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") { t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found) } inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token") inheritedSecret, found := view.Explain(inheritedSecretPath) if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") { t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found) } metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath)) if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") { t.Fatal("metadata disclosed a sensitive provider value") } var effective bytes.Buffer if err := view.WriteEffective(&effective); err != nil { t.Fatalf("write effective configuration: %v", err) } if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") { t.Fatalf("unexpected effective report: %s", effective.String()) } var explanation bytes.Buffer if err := view.WriteExplanation(&explanation, namespacedSecretPath); err != nil || strings.Contains(explanation.String(), "namespaced-private-sentinel") { t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err) } sources := view.Sources() if len(sources) == 0 { t.Fatal("inspection returned no sources") } originalLabel := sources[0].Label sources[0].Label = "changed" if view.Sources()[0].Label != originalLabel { t.Fatal("source mutation reached retained inspection") } model.History[0].After.YAML = "changed" if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" { t.Fatal("explanation mutation reached retained inspection") } history := view.Metadata().History(modelPath) if len(history) == 0 { t.Fatal("metadata history is empty") } history[0].Reasons = []string{"changed"} if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") { t.Fatal("metadata mutation reached retained inspection") } } func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) { _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("invalid backend", map[string]any{ "collections": map[string]any{"private": map[string]any{"backends": []any{ map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}}, }}}, }), }}}) var loadErr *configkit.LoadError if !errors.As(err, &loadErr) { t.Fatalf("invalid backend error = %v, want load error", err) } if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") { t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text) } } func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ Source: configkit.Mapping("main", map[string]any{"version": "2"}), }}}) if err != nil { t.Fatalf("construct engine with accepted declaration: %v", err) } inspection, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } path := configkit.Keys("version") field, found := inspection.Explain(path) if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") { t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found) } var effective bytes.Buffer if err := inspection.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), "version") { t.Fatalf("effective inspection = (%q, %v), want retained version", effective.String(), err) } } type failingInspectionWriter struct{ err error } func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err } func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) { var engine *promptkit.Engine if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("nil engine inspection = %v", err) } var empty promptkit.ConfigurationInspection path := configkit.Keys("profiles", "missing") if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 { t.Fatal("zero inspection reported configuration") } if _, found := empty.Explain(path); found { t.Fatal("zero inspection explained an unknown path") } if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("zero effective writer = %v", err) } if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("zero explanation writer = %v", err) } profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{ "profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}}, })} valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}}, }}) if err != nil { t.Fatalf("construct engine: %v", err) } view, err := valid.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } writeErr := errors.New("report writer failed") writer := failingInspectionWriter{err: writeErr} if err := view.WriteEffective(writer); !errors.Is(err, writeErr) { t.Fatalf("effective writer error = %v", err) } if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) { t.Fatalf("explanation writer error = %v", err) } } func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) { const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n" source := &unreadableAfterConstructionFS{FS: fstest.MapFS{ "config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} inputs, err := promptkit.FilesystemInputs("inspection source", source, "config") if err != nil { t.Fatalf("select inputs: %v", err) } otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{ "config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config") if err != nil { t.Fatalf("select comparison inputs: %v", err) } var modelCalls atomic.Int64 client := countingLLMClient{calls: &modelCalls} endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{ "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct engine: %v", err) } otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{ "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct comparison engine: %v", err) } source.closed.Store(true) otherSource.closed.Store(true) view, err := engine.InspectConfiguration() if err != nil { t.Fatalf("inspect configuration: %v", err) } otherView, err := otherEngine.InspectConfiguration() if err != nil { t.Fatalf("inspect comparison configuration: %v", err) } var expected configkit.Difference for index := 0; index < 3; index++ { differences, err := view.Diff(otherView) if err != nil || len(differences) != 1 { t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err) } if differences[0].Path.String() != "/profiles" || !strings.Contains(differences[0].Left.YAML, model) || !strings.Contains(differences[0].Right.YAML, otherModel) { t.Fatalf("captured difference = %+v", differences[0]) } if index == 0 { expected = differences[0] } else if differences[0].Path.String() != expected.Path.String() || differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML { t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0]) } } if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found { t.Fatal("captured prompt content path was not inspectable") } var report bytes.Buffer if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") { t.Fatalf("write captured report = (%q, %v)", report.String(), err) } if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 { t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } const calls = 24 var wait sync.WaitGroup errs := make(chan error, calls) for index := 0; index < calls; index++ { wait.Add(1) go func(index int) { defer wait.Done() switch index % 3 { case 0: _, err := engine.InspectConfiguration() errs <- err case 1: _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"}) errs <- err case 2: differences, err := view.Diff(otherView) if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() || differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) { err = errors.New("concurrent comparison returned an unexpected difference") } errs <- err } }(index) } wait.Wait() close(errs) for err := range errs { if err != nil { t.Fatal(err) } } if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 { t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load()) } }