401 lines
16 KiB
Go
401 lines
16 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"reflect"
|
|
"testing"
|
|
|
|
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
|
|
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckprompt"
|
|
configkit "gitea.maximumdirect.net/eric/ConfigKit"
|
|
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
func TestComposedApplicationDiffUsesFixturesAndCampaignOverride(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
args := []string{"config", "diff", "--scope", "app", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml"}
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("fixture diff outcome = %#v", outcome)
|
|
}
|
|
value := composedReportValue(t, output.Bytes(), "changes", "default_campaign")
|
|
if value == nil || value.Value != "autumn" {
|
|
t.Fatalf("fixture diff default_campaign = %#v; report %q", value, output.String())
|
|
}
|
|
|
|
output.Reset()
|
|
args = append(args, "--campaign", "spring", "--limit=0")
|
|
outcome = app.Execute(context.Background(), args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("overridden fixture diff outcome = %#v", outcome)
|
|
}
|
|
value = composedReportValue(t, output.Bytes(), "changes", "campaigns", "spring", "model")
|
|
if value == nil || value.Value != "spring-model-after" {
|
|
t.Fatalf("campaign override spring model = %#v; report %q", value, output.String())
|
|
}
|
|
}
|
|
|
|
func TestApplicationCacheSeparatesSelectionsCampaignValuesAndPresence(t *testing.T) {
|
|
cache := &runCache{}
|
|
beforeSelection := ckconfigSelection("fixtures/before.yml")
|
|
afterSelection := ckconfigSelection("fixtures/after.yml")
|
|
load := func(selection ckconfig.Selection, input *inspectionInput) *configkit.Result[appConfiguration] {
|
|
t.Helper()
|
|
result, err := loadApplicationConfig(context.Background(), selection, input, cache)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return result
|
|
}
|
|
|
|
before := load(beforeSelection, nil)
|
|
after := load(afterSelection, nil)
|
|
if before.Config.DefaultCampaign != "spring" || after.Config.DefaultCampaign != "autumn" || before == after {
|
|
t.Fatalf("file selections shared or returned wrong values: before=%#v after=%#v", before.Config, after.Config)
|
|
}
|
|
defaultAfter := load(afterSelection, nil)
|
|
suppliedSpring := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "spring", Set: true}})
|
|
suppliedAutumn := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}})
|
|
suppliedEmpty := load(afterSelection, &inspectionInput{Campaign: commandkit.Supplied[string]{Set: true}})
|
|
if defaultAfter != after || suppliedSpring.Config.DefaultCampaign != "spring" || suppliedAutumn.Config.DefaultCampaign != "autumn" || suppliedEmpty.Config.DefaultCampaign != "" {
|
|
t.Fatalf("campaign loading values = default:%q spring:%q autumn:%q empty:%q", defaultAfter.Config.DefaultCampaign, suppliedSpring.Config.DefaultCampaign, suppliedAutumn.Config.DefaultCampaign, suppliedEmpty.Config.DefaultCampaign)
|
|
}
|
|
if suppliedSpring == suppliedAutumn || suppliedAutumn == defaultAfter || suppliedEmpty == defaultAfter || len(cache.applications) != 5 {
|
|
t.Fatalf("cache did not distinguish selection/input presence: entries=%d", len(cache.applications))
|
|
}
|
|
|
|
uncachedOne, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
uncachedTwo, err := loadApplicationConfig(context.Background(), afterSelection, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if uncachedOne == uncachedTwo {
|
|
t.Fatal("nil cache reused an application result")
|
|
}
|
|
}
|
|
|
|
func TestApplicationCacheDoesNotRetainFailedLoads(t *testing.T) {
|
|
cache := &runCache{}
|
|
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "fixtures/missing.yml", Set: true}}
|
|
for range 2 {
|
|
if _, err := loadApplicationConfig(context.Background(), selection, nil, cache); err == nil {
|
|
t.Fatal("missing configuration unexpectedly loaded")
|
|
}
|
|
}
|
|
if len(cache.applications) != 0 {
|
|
t.Fatalf("failed result was cached: %#v", cache.applications)
|
|
}
|
|
}
|
|
|
|
func TestComposedPromptKitAndDomainCommandsRemainAvailable(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
commands := [][]string{
|
|
{"config", "validate", "--scope", "all", "--campaign", "spring", "--limit=0"},
|
|
{"config", "show", "--scope", "promptkit", "--campaign", "spring"},
|
|
{"config", "sources", "--scope", "promptkit", "--campaign", "spring"},
|
|
{"config", "explain", "--scope", "promptkit", "--path", "/profiles/spring/model", "--campaign", "spring"},
|
|
{"promptkit", "show", "prompt", "briefing", "--campaign", "spring", "--format", "yaml"},
|
|
{"promptkit", "show", "profile", "spring", "--campaign", "spring"},
|
|
{"session", "restore", "session-42", "--campaign", "spring"},
|
|
}
|
|
for _, args := range commands {
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Errorf("command %q outcome = %#v", args, outcome)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestComposedRequirementsDiscoveryUsesSelectedCampaign(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
wantMode bool
|
|
wantModeIssue bool
|
|
}{
|
|
{
|
|
name: "conditional dependency is unresolved without mode",
|
|
args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring"},
|
|
wantModeIssue: true,
|
|
},
|
|
{
|
|
name: "supplied mode discovers conditional notes",
|
|
args: []string{"promptkit", "requirements", "show", "briefing", "--campaign", "spring", "--var", "mode=full", "--format", "yaml"},
|
|
wantMode: true,
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
|
t.Fatalf("requirements outcome = %#v, report %q", outcome, output.String())
|
|
}
|
|
|
|
var report struct {
|
|
PromptID string `yaml:"prompt_id"`
|
|
PromptVersion string `yaml:"prompt_version"`
|
|
ProfileID string `yaml:"profile_id"`
|
|
Complete bool `yaml:"complete"`
|
|
Inputs []struct {
|
|
Name string `yaml:"name"`
|
|
Kind string `yaml:"kind"`
|
|
Declared bool `yaml:"declared"`
|
|
} `yaml:"inputs"`
|
|
Variables []struct {
|
|
Name string `yaml:"name"`
|
|
Supplied bool `yaml:"supplied"`
|
|
} `yaml:"variables"`
|
|
Issues []struct {
|
|
Code string `yaml:"code"`
|
|
InputName string `yaml:"input_name"`
|
|
VariableName string `yaml:"variable_name"`
|
|
} `yaml:"issues"`
|
|
}
|
|
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode requirements report %q: %v", output.String(), err)
|
|
}
|
|
if report.PromptID != "briefing" || report.PromptVersion != "1" || report.ProfileID != "spring" {
|
|
t.Fatalf("requirements identity = %#v", report)
|
|
}
|
|
|
|
foundTopic, foundNotes, foundMode := false, false, false
|
|
for _, input := range report.Inputs {
|
|
switch input.Name {
|
|
case "topic":
|
|
foundTopic = input.Declared && input.Kind == "required"
|
|
case "notes":
|
|
foundNotes = true
|
|
wantKind := "conditional"
|
|
if test.wantMode {
|
|
wantKind = "required"
|
|
}
|
|
if input.Declared || input.Kind != wantKind {
|
|
t.Errorf("notes input classification = %#v; want undeclared %s", input, wantKind)
|
|
}
|
|
}
|
|
}
|
|
for _, variable := range report.Variables {
|
|
if variable.Name == "mode" {
|
|
foundMode = variable.Supplied == test.wantMode
|
|
}
|
|
}
|
|
if !foundTopic || !foundNotes || !foundMode {
|
|
t.Fatalf("requirements inventory topic=%t notes=%t mode=%t; report %#v", foundTopic, foundNotes, foundMode, report)
|
|
}
|
|
if test.wantModeIssue && (len(report.Issues) != 1 || report.Issues[0].Code != "missing_variable" || report.Issues[0].VariableName != "mode") {
|
|
t.Fatalf("missing mode issue = %#v", report.Issues)
|
|
}
|
|
if !test.wantModeIssue && len(report.Issues) != 0 {
|
|
t.Fatalf("requirements issues with mode supplied=%t: %#v", test.wantMode, report.Issues)
|
|
}
|
|
if report.Complete != test.wantMode {
|
|
t.Fatalf("requirements completeness = %t with mode supplied=%t; report %#v", report.Complete, test.wantMode, report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestComposedRequestValidationUsesFileBindings(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, test := range []struct {
|
|
name string
|
|
args []string
|
|
}{
|
|
{
|
|
name: "default mode",
|
|
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt"},
|
|
},
|
|
{
|
|
name: "conditional notes",
|
|
args: []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring", "--input", "topic=fixtures/topic.txt", "--var", "mode=full", "--input", "notes=fixtures/notes.txt", "--format", "yaml"},
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 {
|
|
t.Fatalf("request validation = %#v, report %q", outcome, output.String())
|
|
}
|
|
var report map[string]any
|
|
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatalf("decode validation confirmation %q: %v", output.String(), err)
|
|
}
|
|
if len(report) != 2 || report["prompt_id"] != "briefing" || report["valid"] != true {
|
|
t.Fatalf("validation confirmation = %#v", report)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestComposedRequestValidationRequiresTopicArtifact(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || outcome.Cause == nil || output.Len() != 0 {
|
|
t.Fatalf("missing topic artifact outcome = %#v, report %q", outcome, output.String())
|
|
}
|
|
}
|
|
|
|
func TestBuildValidationRequestDefaultsOnlyAbsentVariables(t *testing.T) {
|
|
input := &inspectionInput{Campaign: commandkit.Supplied[string]{Value: "autumn", Set: true}}
|
|
cache := &runCache{}
|
|
inputs := map[string]promptkit.ArtifactRef{"topic": promptkit.File("fixtures/topic.txt")}
|
|
explicit := map[string]string{"topic": "", "mode": "", "audience": ""}
|
|
seed := ckprompt.ValidationRequest{Request: promptkit.RunRequest{
|
|
PromptID: "briefing",
|
|
PromptVersion: "2",
|
|
ProfileID: "custom-profile",
|
|
SessionID: "session-9",
|
|
Vars: explicit,
|
|
Inputs: inputs,
|
|
}}
|
|
|
|
got, err := buildValidationRequest(context.Background(), seed, input, cache)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.PromptID != seed.Request.PromptID || got.PromptVersion != seed.Request.PromptVersion || got.ProfileID != seed.Request.ProfileID || got.SessionID != seed.Request.SessionID {
|
|
t.Fatalf("request identity/context changed: %#v", got)
|
|
}
|
|
if got.Vars["topic"] != "" || got.Vars["mode"] != "" || got.Vars["audience"] != "" {
|
|
t.Fatalf("explicit empty values were replaced: %#v", got.Vars)
|
|
}
|
|
if !reflect.DeepEqual(got.Inputs, inputs) || !reflect.DeepEqual(seed.Request.Vars, explicit) {
|
|
t.Fatalf("seed bindings changed: built=%#v seed=%#v", got.Inputs, explicit)
|
|
}
|
|
key := applicationCacheKey{selection: seed.Selection, campaign: input.Campaign}
|
|
cached := cache.applications[key]
|
|
if cached == nil {
|
|
t.Fatal("builder did not load application configuration into invocation state")
|
|
}
|
|
if _, err := loadPromptEngine(context.Background(), seed.Selection, input, cache); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cache.applications[key] != cached {
|
|
t.Fatal("engine loader did not reuse the builder's application configuration")
|
|
}
|
|
|
|
missing := ckprompt.ValidationRequest{Request: promptkit.RunRequest{PromptID: "briefing", Vars: map[string]string{"audience": "internal"}}}
|
|
defaulted, err := buildValidationRequest(context.Background(), missing, input, cache)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if defaulted.Vars["topic"] != "autumn campaign planning" || defaulted.Vars["mode"] != "" || defaulted.Vars["audience"] != "internal" {
|
|
t.Fatalf("missing variables were not defaulted while retaining supplied values: %#v", defaulted.Vars)
|
|
}
|
|
if _, ok := missing.Request.Vars["topic"]; ok {
|
|
t.Fatalf("builder modified the seed variable map: %#v", missing.Request.Vars)
|
|
}
|
|
}
|
|
|
|
func TestPromptKitScopeDiffAndCatalogsUseSelectedCampaign(t *testing.T) {
|
|
app, err := newApp()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "fixtures/before.yml", "--against-config", "fixtures/after.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("PromptKit diff outcome = %#v", outcome)
|
|
}
|
|
value := composedReportValue(t, output.Bytes(), "changes", "profiles", "spring", "model")
|
|
if value == nil || value.Value != "spring-model-after" {
|
|
t.Fatalf("PromptKit diff spring model = %#v; report %q", value, output.String())
|
|
}
|
|
|
|
output.Reset()
|
|
outcome = app.Execute(context.Background(), []string{"promptkit", "list", "prompts", "--config", "fixtures/before.yml", "--campaign", "spring"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("prompt catalog outcome = %#v", outcome)
|
|
}
|
|
var prompts []map[string]any
|
|
if err := yaml.Unmarshal(output.Bytes(), &prompts); err != nil {
|
|
t.Fatalf("decode prompt catalog %q: %v", output.String(), err)
|
|
}
|
|
foundPrompt := false
|
|
for _, prompt := range prompts {
|
|
if prompt["id"] == "briefing" && prompt["version"] == "1" {
|
|
foundPrompt = true
|
|
break
|
|
}
|
|
}
|
|
if !foundPrompt {
|
|
t.Fatalf("selected prompt identity missing from catalog: %#v", prompts)
|
|
}
|
|
|
|
output.Reset()
|
|
outcome = app.Execute(context.Background(), []string{"promptkit", "list", "profiles", "--config", "fixtures/after.yml", "--campaign", "autumn", "--format", "yaml"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess {
|
|
t.Fatalf("profile catalog outcome = %#v", outcome)
|
|
}
|
|
var profiles []string
|
|
if err := yaml.Unmarshal(output.Bytes(), &profiles); err != nil {
|
|
t.Fatalf("decode profile catalog %q: %v", output.String(), err)
|
|
}
|
|
foundProfile := false
|
|
for _, profile := range profiles {
|
|
if profile == "autumn" {
|
|
foundProfile = true
|
|
break
|
|
}
|
|
}
|
|
if !foundProfile {
|
|
t.Fatalf("selected profile missing from catalog: %#v", profiles)
|
|
}
|
|
}
|
|
|
|
func composedReportValue(t *testing.T, data []byte, category string, fields ...string) *yaml.Node {
|
|
t.Helper()
|
|
var document yaml.Node
|
|
if err := yaml.Unmarshal(data, &document); err != nil {
|
|
t.Fatalf("decode composed report %q: %v", data, err)
|
|
}
|
|
node := composedField(document.Content[0], category)
|
|
for _, field := range fields {
|
|
node = composedField(node, field)
|
|
}
|
|
return node
|
|
}
|
|
|
|
func composedField(mapping *yaml.Node, name string) *yaml.Node {
|
|
if mapping == nil || mapping.Kind != yaml.MappingNode {
|
|
return nil
|
|
}
|
|
for index := 0; index+1 < len(mapping.Content); index += 2 {
|
|
if mapping.Content[index].Value == name {
|
|
return mapping.Content[index+1]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ckconfigSelection(path string) ckconfig.Selection {
|
|
return ckconfig.Selection{Config: commandkit.Supplied[string]{Value: path, Set: true}}
|
|
}
|