package main import ( "bytes" "context" "io" "reflect" "testing" commandkit "gitea.maximumdirect.net/eric/CommandKit" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig" "gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt" configkit "gitea.maximumdirect.net/eric/ConfigKit" promptkit "gitea.maximumdirect.net/eric/promptkit" "go.yaml.in/yaml/v4" ) func TestComposedApplicationDiffUsesFixturesAndCampaignOverride(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } args := []string{"config", "diff", "--scope", "app", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml"} var output bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("fixture diff outcome = %#v", outcome) } value := composedReportValue(t, output.Bytes(), "changes", "default_campaign") if value == nil || value.Value != "autumn" { t.Fatalf("fixture diff default_campaign = %#v; report %q", value, output.String()) } output.Reset() args = append(args, "--campaign", "spring", "--limit=0") outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("overridden fixture diff outcome = %#v", outcome) } value = composedReportValue(t, output.Bytes(), "changes", "campaigns", "spring", "model") if value == nil || value.Value != "spring-model-after" { t.Fatalf("campaign override spring model = %#v; report %q", value, output.String()) } } func TestApplicationCacheSeparatesSelectionsCampaignValuesAndPresence(t *testing.T) { cache := &runCache{} beforeSelection := ckconfigSelection("fixtures/before.yml") afterSelection := ckconfigSelection("fixtures/after.yml") load := func(selection ckconfig.Selection, input *inspectionInput) *configkit.Result[appConfiguration] { t.Helper() result, err := loadApplicationConfig(context.Background(), selection, input, cache) if err != nil { t.Fatal(err) } return result } before := load(beforeSelection, nil) after := load(afterSelection, nil) if before.Config.DefaultCampaign != "spring" || after.Config.DefaultCampaign != "autumn" || before == after { t.Fatalf("file selections shared or returned wrong values: before=%#v after=%#v", before.Config, after.Config) } defaultAfter := load(afterSelection, nil) suppliedSpring := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "spring", Set: true}}) suppliedAutumn := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}}) suppliedEmpty := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Set: true}}) if defaultAfter != after || suppliedSpring.Config.DefaultCampaign != "spring" || suppliedAutumn.Config.DefaultCampaign != "autumn" || suppliedEmpty.Config.DefaultCampaign != "" { t.Fatalf("campaign loading values = default:%q spring:%q autumn:%q empty:%q", defaultAfter.Config.DefaultCampaign, suppliedSpring.Config.DefaultCampaign, suppliedAutumn.Config.DefaultCampaign, suppliedEmpty.Config.DefaultCampaign) } if suppliedSpring == suppliedAutumn || suppliedAutumn == defaultAfter || suppliedEmpty == defaultAfter || len(cache.applications) != 5 { t.Fatalf("cache did not distinguish selection/input presence: entries=%d", len(cache.applications)) } uncachedOne, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil) if err != nil { t.Fatal(err) } uncachedTwo, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil) if err != nil { t.Fatal(err) } if uncachedOne == uncachedTwo { t.Fatal("nil cache reused an application result") } } func TestApplicationCacheDoesNotRetainFailedLoads(t *testing.T) { cache := &runCache{} selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "fixtures/missing.yml", Set: true}} for range 2 { if _, err := loadApplicationConfig(context.Background(), selection, nil, cache); err == nil { t.Fatal("missing configuration unexpectedly loaded") } } if len(cache.applications) != 0 { t.Fatalf("failed result was cached: %#v", cache.applications) } } func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } commands := [][]string{ {"config", "validate", "--scope", "all", "--campaign", "spring", "--limit=0"}, {"config", "show", "--scope", "promptkit", "--campaign", "spring"}, {"config", "sources", "--scope", "promptkit", "--campaign", "spring"}, {"config", "explain", "--scope", "promptkit", "--path", "/profiles/spring/model", "--campaign", "spring"}, {"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--format", "yaml"}, {"promptkit", "show", "profile", "spring", "--campaign", "spring"}, {"session", "restore", "session-42", "--campaign", "spring"}, } for _, args := range commands { outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Errorf("command %q outcome = %#v", args, outcome) } } } func TestComposedRequirementsDiscoveryUsesSelectedCampaign(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } for _, test := range []struct { name string args []string }{ {name: "without variables", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}}, {name: "with caller variable", args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--var", "mode=full", "--format", "yaml"}}, } { t.Run(test.name, func(t *testing.T) { var output bytes.Buffer outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 { t.Fatalf("requirements outcome = %#v, report %q", outcome, output.String()) } var report map[string]any if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { t.Fatalf("decode requirements report %q: %v", output.String(), err) } for _, key := range []string{"prompt_id", "version", "profile_id", "complete", "inputs", "variables", "issues"} { if _, found := report[key]; !found { t.Errorf("requirements report is missing %q: %#v", key, report) } } if report["prompt_id"] != "briefing" || report["version"] != "1" || report["profile_id"] != "spring" { t.Fatalf("requirements report metadata = %#v", report) } }) } } func TestComposedRequestValidationUsesFileBindings(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } for _, test := range []struct { name string args []string }{ { name: "default mode", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt"}, }, { name: "conditional notes", args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt", "--var", "mode=full", "--input", "notes=fixtures/notes.txt", "--format", "yaml"}, }, } { t.Run(test.name, func(t *testing.T) { var output bytes.Buffer outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 { t.Fatalf("request validation = %#v, report %q", outcome, output.String()) } var report map[string]any if err := yaml.Unmarshal(output.Bytes(), &report); err != nil { t.Fatalf("decode validation confirmation %q: %v", output.String(), err) } if len(report) != 2 || report["prompt_id"] != "briefing" || report["valid"] != true { t.Fatalf("validation confirmation = %#v", report) } }) } } func TestComposedRequestValidationRequiresTopicArtifact(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 { t.Fatalf("missing topic artifact outcome = %#v, report %q", outcome, output.String()) } } func TestBuildValidationRequestDefaultsOnlyAbsentVariables(t *testing.T) { input := &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}} cache := &runCache{} inputs := map[string]promptkit.ArtifactRef{"topic": promptkit.File("fixtures/topic.txt")} explicit := map[string]string{"topic": "", "mode": "", "audience": ""} seed := ckprompt.ValidationRequest{Request: promptkit.RunRequest{ PromptID: "briefing", ProfileID: "custom-profile", SessionID: "session-9", Vars: explicit, Inputs: inputs, }} got, err := buildValidationRequest(context.Background(), seed, input, cache) if err != nil { t.Fatal(err) } if got.PromptID != seed.Request.PromptID || got.ProfileID != seed.Request.ProfileID || got.SessionID != seed.Request.SessionID { t.Fatalf("request identity/context changed: %#v", got) } if got.Vars["topic"] != "" || got.Vars["mode"] != "" || got.Vars["audience"] != "" { t.Fatalf("explicit empty values were replaced: %#v", got.Vars) } if !reflect.DeepEqual(got.Inputs, inputs) || !reflect.DeepEqual(seed.Request.Vars, explicit) { t.Fatalf("seed bindings changed: built=%#v seed=%#v", got.Inputs, explicit) } key := applicationCacheKey{selection: seed.Selection, campaign: input.Campaign} cached := cache.applications[key] if cached == nil { t.Fatal("builder did not load application configuration into invocation state") } if _, err := loadPromptEngine(context.Background(), seed.Selection, input, cache); err != nil { t.Fatal(err) } if cache.applications[key] != cached { t.Fatal("engine loader did not reuse the builder's application configuration") } missing := ckprompt.ValidationRequest{Request: promptkit.RunRequest{PromptID: "briefing", Vars: map[string]string{"audience": "internal"}}} defaulted, err := buildValidationRequest(context.Background(), missing, input, cache) if err != nil { t.Fatal(err) } if defaulted.Vars["topic"] != "autumn campaign planning" || defaulted.Vars["mode"] != "" || defaulted.Vars["audience"] != "internal" { t.Fatalf("missing variables were not defaulted while retaining supplied values: %#v", defaulted.Vars) } if _, ok := missing.Request.Vars["topic"]; ok { t.Fatalf("builder modified the seed variable map: %#v", missing.Request.Vars) } } func TestPromptKitScopeDiffAndCatalogsUseSelectedCampaign(t *testing.T) { app, err := newApp() if err != nil { t.Fatal(err) } var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("PromptKit diff outcome = %#v", outcome) } value := composedReportValue(t, output.Bytes(), "changes", "profiles", "spring", "model") if value == nil || value.Value != "spring-model-after" { t.Fatalf("PromptKit diff spring model = %#v; report %q", value, output.String()) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--config", "fixtures/before.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("prompt catalog outcome = %#v", outcome) } var prompts []map[string]any if err := yaml.Unmarshal(output.Bytes(), &prompts); err != nil { t.Fatalf("decode prompt catalog %q: %v", output.String(), err) } foundPrompt := false for _, prompt := range prompts { if prompt["id"] == "briefing" && prompt["version"] == "1" { foundPrompt = true break } } if !foundPrompt { t.Fatalf("selected prompt identity missing from catalog: %#v", prompts) } output.Reset() outcome = app.Execute(context.Background(), []string{"promptkit", "list", "profiles", "--config", "fixtures/after.yml", "--campaign", "autumn", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Fatalf("profile catalog outcome = %#v", outcome) } var profiles []string if err := yaml.Unmarshal(output.Bytes(), &profiles); err != nil { t.Fatalf("decode profile catalog %q: %v", output.String(), err) } foundProfile := false for _, profile := range profiles { if profile == "autumn" { foundProfile = true break } } if !foundProfile { t.Fatalf("selected profile missing from catalog: %#v", profiles) } } func TestComposedCatalogsAndSmokeUseInjectedClientOnlyForInference(t *testing.T) { calls := 0 app, err := newAppWithLLMClient(&exampleLLMClient{calls: &calls}) if err != nil { t.Fatal(err) } commands := [][]string{ {"promptkit", "list", "backends", "--campaign", "spring"}, {"promptkit", "list", "namespaces", "--campaign", "spring"}, {"promptkit", "list", "collections", "--campaign", "spring"}, {"promptkit", "list", "prompts", "--namespace", "field-notes", "--campaign", "spring"}, {"promptkit", "show", "prompt", "briefing", "--campaign", "spring"}, {"promptkit", "requirements", "show", "briefing", "--campaign", "spring"}, } for _, args := range commands { var output bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard}) if outcome.Kind != commandkit.OutcomeSuccess { t.Errorf("metadata command %q outcome = %#v; output %q", args, outcome, output.String()) } if output.Len() == 0 { t.Errorf("metadata command %q produced no report", args) } } if calls != 0 { t.Fatalf("metadata and requirements commands invoked the LLM client %d times", calls) } var output bytes.Buffer outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", "--campaign", "spring", "--prompt", "briefing", "--profile", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard}) if output.Len() == 0 || !bytes.Contains(output.Bytes(), []byte("tests:")) { t.Fatalf("aggregate smoke report missing: outcome=%#v output=%q", outcome, output.String()) } if calls == 0 { t.Fatalf("smoke command did not use the injected LLM client: outcome=%#v output=%q", outcome, output.String()) } } func composedReportValue(t *testing.T, data []byte, category string, fields ...string) *yaml.Node { t.Helper() var document yaml.Node if err := yaml.Unmarshal(data, &document); err != nil { t.Fatalf("decode composed report %q: %v", data, err) } node := composedField(document.Content[0], category) for _, field := range fields { node = composedField(node, field) } return node } func composedField(mapping *yaml.Node, name string) *yaml.Node { if mapping == nil || mapping.Kind != yaml.MappingNode { return nil } for index := 0; index+1 < len(mapping.Content); index += 2 { if mapping.Content[index].Value == name { return mapping.Content[index+1] } } return nil } func ckconfigSelection(path string) ckconfig.Selection { return ckconfig.Selection{Config: commandkit.Supplied[string]{Value: path, Set: true}} }