diff --git a/configuration_diff_contract_test.go b/configuration_diff_contract_test.go index f5f05cc..a560c7b 100644 --- a/configuration_diff_contract_test.go +++ b/configuration_diff_contract_test.go @@ -21,7 +21,7 @@ func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) { t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences) } difference := differences[0] - if got, want := difference.Path.String(), "/profiles/offline/model"; got != want { + if got, want := difference.Path.String(), "/profiles"; got != want { t.Fatalf("difference path = %q, want %q", got, want) } if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") { @@ -94,18 +94,14 @@ func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) { if err != nil { t.Fatalf("compare presence and sequence values: %v", err) } - if len(differences) != 2 { - t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences) + if len(differences) != 1 || differences[0].Path.String() != "/backends" { + t.Fatalf("atomic backend list difference = %+v", differences) } - byPath := differencesByPath(differences) - nullable, ok := byPath["/backends/local/extra_params/nullable"] - if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" { - t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok) - } - sequence, ok := byPath["/backends/local/extra_params/sequence"] - if !ok || !sequence.Left.Present || !sequence.Right.Present || - !strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") { - t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok) + difference := differences[0] + if !difference.Left.Present || !difference.Right.Present || + strings.Contains(difference.Left.YAML, "nullable:") || !strings.Contains(difference.Right.YAML, "nullable: null") || + !strings.Contains(difference.Left.YAML, "earlier") || !strings.Contains(difference.Right.YAML, "later") { + t.Fatalf("atomic sequence snapshots = %+v", difference) } } @@ -123,22 +119,19 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t if err != nil { t.Fatalf("compare sensitive values: %v", err) } - if len(differences) != 2 { - t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences) + if len(differences) != 1 || differences[0].Path.String() != "/backends" { + t.Fatalf("sensitive backend list difference = %+v", differences) } - byPath := differencesByPath(differences) - secret, ok := byPath["/backends/local/extra_params/token"] - if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged { - t.Fatalf("sensitive difference = (%+v, %t)", secret, ok) + secret := differences[0] + if !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged { + t.Fatalf("sensitive difference = %+v", secret) } if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") || strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") { t.Fatalf("sensitive snapshots disclosed a value: %+v", secret) } - visible, ok := byPath["/backends/local/extra_params/visible"] - if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged || - !strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") { - t.Fatalf("public sibling difference = (%+v, %t)", visible, ok) + if !strings.Contains(secret.Left.YAML, "before-value") || !strings.Contains(secret.Right.YAML, "after-value") { + t.Fatalf("public sibling missing from snapshots: %+v", secret) } differences[0].Path = configkit.Keys("caller", "mutation") @@ -148,18 +141,15 @@ func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t if err != nil { t.Fatalf("repeat comparison after mutating returned records: %v", err) } - if len(again) != 2 { - t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again) + if len(again) != 1 { + t.Fatalf("repeat difference count = %d, want 1: %+v", len(again), again) } againByPath := differencesByPath(again) if _, exists := againByPath["/caller/mutation"]; exists { t.Fatal("mutating returned path reached retained inspection") } - if _, exists := againByPath["/backends/local/extra_params/token"]; !exists { - t.Fatalf("original sensitive path was lost after mutation: %+v", again) - } - if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists { - t.Fatalf("original public path was lost after mutation: %+v", again) + if _, exists := againByPath["/backends"]; !exists { + t.Fatalf("original backend path was lost after mutation: %+v", again) } for _, difference := range again { if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" { @@ -183,7 +173,7 @@ func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) if err != nil { t.Fatalf("compare sensitive shape replacement: %v", err) } - if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" { + if len(differences) != 1 || differences[0].Path.String() != "/backends" { t.Fatalf("shape replacement differences = %+v", differences) } difference := differences[0] @@ -202,16 +192,16 @@ func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationIns func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection { t.Helper() - backend := map[string]any{"endpoint": "https://example.invalid/v1"} + backend := map[string]any{"id": "local", "endpoint": "https://example.invalid/v1"} if extraParams != nil { backend["extra_params"] = extraParams } engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{ - "backends": map[string]any{"local": backend}, - "profiles": map[string]any{"offline": map[string]any{ - "backend": "local", - "model": model, + "backends": []any{backend}, + "profiles": []any{map[string]any{ + "id": "offline", "backend": "local", + "model": model, }}, })}}}}, }}) diff --git a/configuration_inspection_contract_test.go b/configuration_inspection_contract_test.go index 931be8e..14d55b1 100644 --- a/configuration_inspection_contract_test.go +++ b/configuration_inspection_contract_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "strings" "sync" @@ -19,23 +20,26 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing. engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{ {Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{ - "backends": map[string]any{"local": map[string]any{ - "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY", + "backends": []any{map[string]any{ + "id": "local", "endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY", "extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"}, }}, "collections": map[string]any{"literal": map[string]any{ - "backends": map[string]any{"local": map[string]any{ - "endpoint": "https://example.invalid/v1", - "extra_params": map[string]any{"token": "namespaced-private-sentinel"}, - }}, - "profiles": map[string]any{ - "base.v1": map[string]any{"backend": "local", "model": "default-model"}, - "id": map[string]any{"extends": "base.v1"}, + "backends": []any{ + map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "namespaced-private-sentinel"}}, + map[string]any{"id": "child", "extends": map[string]any{"id": "local"}}, + }, + "profiles": []any{ + map[string]any{"id": "base.v1", "backend": "local", "model": "default-model"}, + map[string]any{"id": "id", "extends": map[string]any{"id": "base.v1"}}, }, }}, })}}}, {Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{ - "collections": map[string]any{"literal": map[string]any{"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}}}}, + "collections": map[string]any{"literal": map[string]any{ + "backends": []any{map[string]any{"id": "local", "endpoint": "https://example.invalid/v1", "extra_params": map[string]any{"token": "replacement-private-sentinel"}}}, + "profiles": []any{map[string]any{"id": "base.v1", "backend": "local", "model": "operator-model"}}, + }}, })}}}, }, }}) @@ -46,29 +50,38 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing. if err != nil { t.Fatalf("inspect configuration: %v", err) } - modelPath := configkit.Keys("collections", "literal", "profiles", "base.v1", "model") + modelPath := configkit.Keys("collections", "literal", "profiles").Index(0).Key("model") model, found := view.Explain(modelPath) - if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 { + if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) == 0 { t.Fatalf("layered model explanation = (%+v, %t)", model, found) } - inheritedPath := configkit.Keys("collections", "literal", "profiles", "id", "model") + inheritedPath := configkit.Keys("collections", "literal", "profiles").Index(1).Key("model") inherited, found := view.Explain(inheritedPath) if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) { t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found) } - if inheritedPath.String() != "/collections/literal/profiles/id/model" { + if inheritedPath.String() != "/collections/literal/profiles/1/model" { t.Fatalf("literal-key path = %q", inheritedPath) } - secretPath := configkit.Keys("backends", "local", "extra_params", "token") + secretPath := configkit.Keys("backends").Index(0).Key("extra_params").Key("token") secret, found := view.Explain(secretPath) if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") { t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found) } - namespacedSecretPath := configkit.Keys("collections", "literal", "backends", "local", "extra_params", "token") + namespacedSecretPath := configkit.Keys("collections", "literal", "backends").Index(0).Key("extra_params").Key("token") namespacedSecret, found := view.Explain(namespacedSecretPath) if !found || !namespacedSecret.Effective.Redacted || strings.Contains(namespacedSecret.Effective.YAML, "namespaced-private-sentinel") { t.Fatalf("namespaced provider value was not redacted: (%+v, %t)", namespacedSecret, found) } + inheritedSecretPath := configkit.Keys("collections", "literal", "backends").Index(1).Key("extra_params").Key("token") + inheritedSecret, found := view.Explain(inheritedSecretPath) + if !found || !inheritedSecret.Effective.Redacted || strings.Contains(inheritedSecret.Effective.YAML, "replacement-private-sentinel") { + t.Fatalf("inherited provider value was not redacted: (%+v, %t)", inheritedSecret, found) + } + metadata := fmt.Sprintf("%+v", view.Metadata().History(inheritedSecretPath)) + if strings.Contains(metadata, "replacement-private-sentinel") || strings.Contains(metadata, "namespaced-private-sentinel") { + t.Fatal("metadata disclosed a sensitive provider value") + } var effective bytes.Buffer if err := view.WriteEffective(&effective); err != nil { t.Fatalf("write effective configuration: %v", err) @@ -104,9 +117,26 @@ func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing. } } +func TestConfigurationInspectionFailureDoesNotDiscloseSensitiveValues(t *testing.T) { + _, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("invalid backend", map[string]any{ + "collections": map[string]any{"private": map[string]any{"backends": []any{ + map[string]any{"id": "backend", "endpoint": "invalid endpoint", "extra_params": map[string]any{"token": "failure-private-sentinel"}}, + }}}, + }), + }}}) + var loadErr *configkit.LoadError + if !errors.As(err, &loadErr) { + t.Fatalf("invalid backend error = %v, want load error", err) + } + if text := fmt.Sprintf("%v %+v", err, loadErr.Diagnostics()); strings.Contains(text, "failure-private-sentinel") { + t.Fatalf("failure diagnostics disclosed a sensitive value: %s", text) + } +} + func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ - Source: configkit.Mapping("main", map[string]any{"version": "1"}), + Source: configkit.Mapping("main", map[string]any{"version": "2"}), }}}) if err != nil { t.Fatalf("construct engine with accepted declaration: %v", err) @@ -117,7 +147,7 @@ func TestConfigurationInspectionRetainsMainSpecificationDeclaration(t *testing.T } path := configkit.Keys("version") field, found := inspection.Explain(path) - if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "1") { + if !found || !inspection.Metadata().IsExplicit(path) || !strings.Contains(field.Effective.YAML, "2") { t.Fatalf("main specification declaration was not retained: (%+v, %t)", field, found) } var effective bytes.Buffer @@ -150,9 +180,8 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) { t.Fatalf("zero explanation writer = %v", err) } - endpoint, model := "https://configured.example/v1", "configured-model" - profile := configkit.Input{Source: promptkit.ProfileSource("inspection test profile", map[string]promptkit.ProfileConfig{ - "inspection-profile": {Endpoint: &endpoint, Model: &model}, + profile := configkit.Input{Source: configkit.Mapping("inspection test profile", map[string]any{ + "profiles": []any{map[string]any{"id": "inspection-profile", "endpoint": "https://configured.example/v1", "model": "configured-model"}}, })} valid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ Underlays: []configkit.Underlay{{Inputs: []configkit.Input{profile}}}, @@ -169,14 +198,15 @@ func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) { if err := view.WriteEffective(writer); !errors.Is(err, writeErr) { t.Fatalf("effective writer error = %v", err) } - if err := view.WriteExplanation(writer, configkit.Keys("profiles", "inspection-profile", "model")); !errors.Is(err, writeErr) { + if err := view.WriteExplanation(writer, configkit.Keys("profiles").Index(0).Key("model")); !errors.Is(err, writeErr) { t.Fatalf("explanation writer error = %v", err) } } func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) { + const promptDocument = "prompts:\n - id: captured\n default_profile: offline\n messages:\n - role: user\n content_file: body.tmpl\n output:\n format: text\n validation_mode: none\n" source := &unreadableAfterConstructionFS{FS: fstest.MapFS{ - "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, + "config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} inputs, err := promptkit.FilesystemInputs("inspection source", source, "config") @@ -184,7 +214,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi t.Fatalf("select inputs: %v", err) } otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{ - "config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)}, + "config/prompt.yaml": &fstest.MapFile{Data: []byte(promptDocument)}, "config/body.tmpl": &fstest.MapFile{Data: []byte("hello")}, }} otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config") @@ -195,16 +225,16 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi client := countingLLMClient{calls: &modelCalls} endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model" engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{ - "offline": {Endpoint: &endpoint, Model: &model}, + Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("offline profile", map[string]any{ + "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": model}}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct engine: %v", err) } otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ - Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{ - "offline": {Endpoint: &endpoint, Model: &otherModel}, + Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: configkit.Mapping("comparison profile", map[string]any{ + "profiles": []any{map[string]any{"id": "offline", "endpoint": endpoint, "model": otherModel}}, })}}}}, }}, promptkit.WithLLMClient(client)) if err != nil { @@ -226,7 +256,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi if err != nil || len(differences) != 1 { t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err) } - if differences[0].Path.String() != "/profiles/offline/model" || + if differences[0].Path.String() != "/profiles" || !strings.Contains(differences[0].Left.YAML, model) || !strings.Contains(differences[0].Right.YAML, otherModel) { t.Fatalf("captured difference = %+v", differences[0]) @@ -238,7 +268,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0]) } } - if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found { + if _, found := view.Explain(configkit.Keys("prompts").Index(0).Key("messages").Index(0).Key("content_file")); !found { t.Fatal("captured prompt content path was not inspectable") } var report bytes.Buffer diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 21b0755..1b63465 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -359,6 +359,14 @@ Validation: converted inspection/diff tests find the expected values/origins and do not disclose synthetic credentials. Relax only assertions tied to deliberate upstream history/diagnostic changes, not meaningful redaction expectations. +Completion: sensitivity matching and inspection paths now recognize resource +list indexes. Public inspection and diff fixtures use identified lists and +ConfigKit's atomic sequence comparison, including replaced and inherited +sensitive values, metadata, failure diagnostics, and detached reports. Validation +passed: `go test . -run '^TestConfigurationInspection' -count=1` with Go 1.27.0, +formatting, and `git diff --check`. Internal package tests remain deferred until +the remaining map-indexed loader fixtures are converted. + ## Stage 8: Convert Public Source Builders To Explicit-ID Slices Scope: `profiles.go`, `backends.go`, source-builder GoDoc, and owning construction diff --git a/internal/configuration/inspection.go b/internal/configuration/inspection.go index 80368b0..efa747d 100644 --- a/internal/configuration/inspection.go +++ b/internal/configuration/inspection.go @@ -2,6 +2,7 @@ package configuration import ( "io" + "strconv" "strings" "go.maximumdirect.net/configkit" @@ -66,11 +67,16 @@ func secretProviderPath(path configkit.Path) bool { } func providerExtraParamsPath(parts []string) bool { - if len(parts) >= 4 && parts[1] != "" && + if len(parts) >= 4 && sequenceIndex(parts[1]) && (parts[0] == "backends" || parts[0] == "profiles") && parts[2] == "extra_params" { return true } return len(parts) >= 6 && parts[0] == "collections" && parts[1] != "" && (parts[2] == "backends" || parts[2] == "profiles") && - parts[3] != "" && parts[4] == "extra_params" + sequenceIndex(parts[3]) && parts[4] == "extra_params" +} + +func sequenceIndex(segment string) bool { + index, err := strconv.Atoi(segment) + return err == nil && index >= 0 && strconv.Itoa(index) == segment } diff --git a/internal/configuration/inspection_test.go b/internal/configuration/inspection_test.go index 2c71d70..3f40552 100644 --- a/internal/configuration/inspection_test.go +++ b/internal/configuration/inspection_test.go @@ -12,15 +12,15 @@ func TestSecretProviderPathRecognizesResourceExtraParams(t *testing.T) { path configkit.Path want bool }{ - {name: "backend nested secret", path: configkit.Keys("backends", "provider", "extra_params", "nested", "Token"), want: true}, - {name: "profile nested secret", path: configkit.Keys("profiles", "assistant", "extra_params", "authorization"), want: true}, - {name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends", "default", "extra_params", "nested", "Token"), want: true}, - {name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles", "model", "extra_params", "authorization"), want: true}, + {name: "backend nested secret", path: configkit.Keys("backends").Index(0).Key("extra_params").Key("nested").Key("Token"), want: true}, + {name: "profile nested secret", path: configkit.Keys("profiles").Index(1).Key("extra_params").Key("authorization"), want: true}, + {name: "collection backend nested secret", path: configkit.Keys("collections", "provider", "backends").Index(2).Key("extra_params").Key("nested").Key("Token"), want: true}, + {name: "collection profile secret", path: configkit.Keys("collections", "assistant", "profiles").Index(0).Key("extra_params").Key("authorization"), want: true}, {name: "unrelated nested mapping", path: configkit.Keys("application", "backends", "provider", "extra_params", "token"), want: false}, {name: "extra params outside resource map", path: configkit.Keys("backends", "provider", "options", "extra_params", "token"), want: false}, {name: "collection extra params outside resource map", path: configkit.Keys("collections", "provider", "settings", "backends", "default", "extra_params", "token"), want: false}, {name: "missing resource key", path: configkit.Keys("backends", "", "extra_params", "token"), want: false}, - {name: "ordinary provider value", path: configkit.Keys("profiles", "assistant", "extra_params", "provider", "mode"), want: false}, + {name: "ordinary provider value", path: configkit.Keys("profiles").Index(0).Key("extra_params").Key("provider").Key("mode"), want: false}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) {