promptkit/configuration_inspection_contract_test.go

package promptkit_test

import (
	"bytes"
	"context"
	"errors"
	"io"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/ConfigKit"
	"gitea.maximumdirect.net/eric/promptkit"
)

func TestConfigurationInspectionReportsLiteralKeysAndRedactedHistory(t *testing.T) {
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{
			{Inputs: []configkit.Input{{Source: configkit.Mapping("defaults", map[string]any{
				"backends": map[string]any{"local": map[string]any{
					"endpoint": "https://example.invalid/v1", "api_key_env": "OPTIONAL_KEY",
					"extra_params": map[string]any{"token": "private-sentinel", "visible": "public-value"},
				}},
				"profiles": map[string]any{
					"base.v1":    map[string]any{"backend": "local", "model": "default-model"},
					"literal/id": map[string]any{"extends": "base.v1"},
				},
			})}}},
			{Inputs: []configkit.Input{{Source: configkit.Mapping("operator", map[string]any{
				"profiles": map[string]any{"base.v1": map[string]any{"model": "operator-model"}},
			})}}},
		},
	}})
	if err != nil {
		t.Fatalf("construct layered engine: %v", err)
	}
	view, err := engine.InspectConfiguration()
	if err != nil {
		t.Fatalf("inspect configuration: %v", err)
	}
	modelPath := configkit.Keys("profiles", "base.v1", "model")
	model, found := view.Explain(modelPath)
	if !found || !strings.Contains(model.Effective.YAML, "operator-model") || len(model.History) < 2 {
		t.Fatalf("layered model explanation = (%+v, %t)", model, found)
	}
	inheritedPath := configkit.Keys("profiles", "literal/id", "model")
	inherited, found := view.Explain(inheritedPath)
	if !found || !strings.Contains(inherited.Effective.YAML, "operator-model") || len(inherited.History) == 0 || !view.Metadata().IsExplicit(inheritedPath) {
		t.Fatalf("literal-key inheritance explanation = (%+v, %t)", inherited, found)
	}
	if inheritedPath.String() != "/profiles/literal~1id/model" {
		t.Fatalf("literal-key path = %q", inheritedPath)
	}
	secretPath := configkit.Keys("backends", "local", "extra_params", "token")
	secret, found := view.Explain(secretPath)
	if !found || !secret.Effective.Redacted || strings.Contains(secret.Effective.YAML, "private-sentinel") {
		t.Fatalf("credential-shaped provider value was not redacted: (%+v, %t)", secret, found)
	}
	var effective bytes.Buffer
	if err := view.WriteEffective(&effective); err != nil {
		t.Fatalf("write effective configuration: %v", err)
	}
	if strings.Contains(effective.String(), "private-sentinel") || !strings.Contains(effective.String(), "OPTIONAL_KEY") || !strings.Contains(effective.String(), "public-value") {
		t.Fatalf("unexpected effective report: %s", effective.String())
	}
	var explanation bytes.Buffer
	if err := view.WriteExplanation(&explanation, secretPath); err != nil || strings.Contains(explanation.String(), "private-sentinel") {
		t.Fatalf("secret explanation = (%q, %v)", explanation.String(), err)
	}

	sources := view.Sources()
	if len(sources) == 0 {
		t.Fatal("inspection returned no sources")
	}
	originalLabel := sources[0].Label
	sources[0].Label = "changed"
	if view.Sources()[0].Label != originalLabel {
		t.Fatal("source mutation reached retained inspection")
	}
	model.History[0].After.YAML = "changed"
	if again, ok := view.Explain(modelPath); !ok || again.History[0].After.YAML == "changed" {
		t.Fatal("explanation mutation reached retained inspection")
	}
	history := view.Metadata().History(modelPath)
	if len(history) == 0 {
		t.Fatal("metadata history is empty")
	}
	history[0].Reasons = []string{"changed"}
	if again := view.Metadata().History(modelPath); len(again) == 0 || (len(again[0].Reasons) > 0 && again[0].Reasons[0] == "changed") {
		t.Fatal("metadata mutation reached retained inspection")
	}
}

type failingInspectionWriter struct{ err error }

func (writer failingInspectionWriter) Write([]byte) (int, error) { return 0, writer.err }

func TestConfigurationInspectionZeroAndWriterErrors(t *testing.T) {
	var engine *promptkit.Engine
	if _, err := engine.InspectConfiguration(); !errors.Is(err, promptkit.ErrInvalidConfig) {
		t.Fatalf("nil engine inspection = %v", err)
	}
	var empty promptkit.ConfigurationInspection
	path := configkit.Keys("profiles", "missing")
	if _, found := empty.Metadata().Lookup(path); found || len(empty.Sources()) != 0 || len(empty.Diagnostics()) != 0 {
		t.Fatal("zero inspection reported configuration")
	}
	if _, found := empty.Explain(path); found {
		t.Fatal("zero inspection explained an unknown path")
	}
	if err := empty.WriteEffective(io.Discard); !errors.Is(err, promptkit.ErrInvalidConfig) {
		t.Fatalf("zero effective writer = %v", err)
	}
	if err := empty.WriteExplanation(io.Discard, path); !errors.Is(err, promptkit.ErrInvalidConfig) {
		t.Fatalf("zero explanation writer = %v", err)
	}

	valid, err := promptkit.NewEngine(promptkit.Config{})
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}
	view, err := valid.InspectConfiguration()
	if err != nil {
		t.Fatalf("inspect configuration: %v", err)
	}
	writeErr := errors.New("report writer failed")
	writer := failingInspectionWriter{err: writeErr}
	if err := view.WriteEffective(writer); !errors.Is(err, writeErr) {
		t.Fatalf("effective writer error = %v", err)
	}
	if err := view.WriteExplanation(writer, configkit.Keys("profiles", "mistral-small-3", "model")); !errors.Is(err, writeErr) {
		t.Fatalf("explanation writer error = %v", err)
	}
}

func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testing.T) {
	source := &unreadableAfterConstructionFS{FS: fstest.MapFS{
		"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
		"config/body.tmpl":   &fstest.MapFile{Data: []byte("hello")},
	}}
	inputs, err := promptkit.FilesystemInputs("inspection source", source, "config")
	if err != nil {
		t.Fatalf("select inputs: %v", err)
	}
	otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
		"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
		"config/body.tmpl":   &fstest.MapFile{Data: []byte("hello")},
	}}
	otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
	if err != nil {
		t.Fatalf("select comparison inputs: %v", err)
	}
	var modelCalls atomic.Int64
	client := countingLLMClient{calls: &modelCalls}
	endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
	engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
			"offline": {Endpoint: &endpoint, Model: &model},
		})}}}},
	}}, promptkit.WithLLMClient(client))
	if err != nil {
		t.Fatalf("construct engine: %v", err)
	}
	otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
		Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
			"offline": {Endpoint: &endpoint, Model: &otherModel},
		})}}}},
	}}, promptkit.WithLLMClient(client))
	if err != nil {
		t.Fatalf("construct comparison engine: %v", err)
	}
	source.closed.Store(true)
	otherSource.closed.Store(true)
	view, err := engine.InspectConfiguration()
	if err != nil {
		t.Fatalf("inspect configuration: %v", err)
	}
	otherView, err := otherEngine.InspectConfiguration()
	if err != nil {
		t.Fatalf("inspect comparison configuration: %v", err)
	}
	var expected configkit.Difference
	for index := 0; index < 3; index++ {
		differences, err := view.Diff(otherView)
		if err != nil || len(differences) != 1 {
			t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
		}
		if differences[0].Path.String() != "/profiles/offline/model" ||
			!strings.Contains(differences[0].Left.YAML, model) ||
			!strings.Contains(differences[0].Right.YAML, otherModel) {
			t.Fatalf("captured difference = %+v", differences[0])
		}
		if index == 0 {
			expected = differences[0]
		} else if differences[0].Path.String() != expected.Path.String() ||
			differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
			t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
		}
	}
	if _, found := view.Explain(configkit.Keys("prompts", "captured", "messages").Index(0).Key("content_file")); !found {
		t.Fatal("captured prompt content path was not inspectable")
	}
	var report bytes.Buffer
	if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
		t.Fatalf("write captured report = (%q, %v)", report.String(), err)
	}
	if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
		t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
	}

	const calls = 24
	var wait sync.WaitGroup
	errs := make(chan error, calls)
	for index := 0; index < calls; index++ {
		wait.Add(1)
		go func(index int) {
			defer wait.Done()
			switch index % 3 {
			case 0:
				_, err := engine.InspectConfiguration()
				errs <- err
			case 1:
				_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
				errs <- err
			case 2:
				differences, err := view.Diff(otherView)
				if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
					differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
					err = errors.New("concurrent comparison returned an unexpected difference")
				}
				errs <- err
			}
		}(index)
	}
	wait.Wait()
	close(errs)
	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
	if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
		t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
	}
}