package cli import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "reflect" "strings" "sync/atomic" "testing" "go.maximumdirect.net/ckconfig" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) func nativePromptFixture(t *testing.T, endpoint string) { t.Helper() t.Chdir(t.TempDir()) writeNativeFile(t, "app.yml", "promptkit:\n configuration: framework.yml\n schema_dir: nonexistent-schemas\ntransient_retries: 0\n") writeNativeFile(t, "framework.yml", fmt.Sprintf(`collections: support: release_version: '1' backends: - id: local endpoint: %q profiles: - id: writer backend: support/local model: controlled-model temperature: 0.8 max_tokens: 99 top_p: 0.9 timeout_seconds: 4 reasoning_effort: high extra_params: stale: true - id: protected backend: support/local model: controlled-model api_key_required: true prompts: - id: summary default_profile: support/protected session_id: '{{.session_variable}}' inputs: - name: transcript required: true content_type: text/plain description: Exact transcript binding - name: glossary required: false description: '' messages: - role: user content: 'template-canary {{input "transcript"}} {{input "unlisted"}} {{if .mode}}{{input "glossary"}}{{end}} {{.title}}' output: format: text validation_mode: none - id: dynamic default_profile: support/protected messages: - role: user content: '{{input .source}}' output: format: text validation_mode: none - id: unsupported default_profile: support/protected messages: - role: user content: '{{define "child"}}{{input "called"}}{{end}}{{template "child" .}}' output: format: text validation_mode: none - id: schema-needed default_profile: support/protected messages: - role: user content: 'Return JSON' output: format: json validation_mode: json_schema schema_path: missing.json review: release_version: '1' profiles: - id: writer backend: support/local model: alternate-model prompts: - id: summary default_profile: review/writer inputs: - name: memo required: true messages: - role: user content: '{{input "memo"}}' output: format: text validation_mode: none `, endpoint)) } type consumerRequirements struct { PromptID string `json:"prompt_id"` ProfileID string `json:"profile_id"` Inputs []struct { Name string `json:"name"` Kind promptkit.RequirementKind `json:"kind"` Declared bool `json:"declared"` ContentType *string `json:"content_type"` Description *string `json:"description"` } `json:"inputs"` Variables []struct { Name string `json:"name"` Supplied bool `json:"supplied"` } `json:"variables"` Complete bool `json:"complete"` Issues []struct { Code promptkit.RequirementIssueCode `json:"code"` } `json:"issues"` } func discoverNative(t *testing.T, prompt string, flags ...string) consumerRequirements { t.Helper() args := []string{"promptkit", "requirements", "show", prompt, "--config=app.yml", "--format=json"} args = append(args, flags...) outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 { t.Fatalf("discovery: %+v %q %q", outcome, output, diagnostics) } var report consumerRequirements if err := json.Unmarshal([]byte(output), &report); err != nil { t.Fatal(err) } for _, excluded := range []string{"template-canary", "unrelated-variable-value", "schema_version"} { if strings.Contains(output, excluded) { t.Fatalf("unexpected discovery content %q: %s", excluded, output) } } return report } func TestRequirementsExposeExactBindingsBeforeArtifactsExist(t *testing.T) { nativePromptFixture(t, "http://127.0.0.1:1/v1") report := discoverNative(t, "support/summary", "--var=mode=", "--var=unrelated=unrelated-variable-value", "--session-id=session") if !report.Complete || report.PromptID != "support/summary" || report.ProfileID != "support/protected" { t.Fatalf("discovery identity/completeness: %+v", report) } var names []string for _, input := range report.Inputs { names = append(names, input.Name) switch input.Name { case "transcript": if input.Kind != promptkit.RequirementRequired || !input.Declared || input.ContentType == nil || *input.ContentType != "text/plain" || input.Description == nil { t.Fatalf("required metadata: %+v", input) } case "unlisted": if input.Kind != promptkit.RequirementRequired || input.Declared { t.Fatalf("undeclared reference: %+v", input) } case "glossary": if input.Kind != promptkit.RequirementOptional || input.Description == nil || *input.Description != "" { t.Fatalf("optional metadata: %+v", input) } } } if !reflect.DeepEqual(names, []string{"glossary", "transcript", "unlisted"}) { t.Fatalf("exact input names: %v", names) } var emptySupplied, absent bool for _, variable := range report.Variables { if variable.Name == "mode" { emptySupplied = variable.Supplied } if variable.Name == "title" { absent = !variable.Supplied } } if !emptySupplied || !absent { t.Fatalf("variable presence: %+v", report.Variables) } schemaIndependent := discoverNative(t, "support/schema-needed") if !schemaIndependent.Complete || len(schemaIndependent.Inputs) != 0 { t.Fatalf("schema-free discovery: %+v", schemaIndependent) } // Complete discovery does not assert that these required bindings exist. outcome, _, diagnostics := executeNative(t, "promptkit", "requirements", "validate", "support/summary", "--config=app.yml", "--profile=support/writer", "--var=mode=", "--var=title=Title", "--session-id=session") if outcome.Status == 0 || diagnostics == "" { t.Fatalf("unbound request validated: %+v %q", outcome, diagnostics) } outcome, _, diagnostics = executeNative(t, "promptkit", "show", "prepared", "support/schema-needed", "--config=app.yml", "--profile=support/writer") if outcome.Status == 0 || diagnostics == "" { t.Fatalf("missing preparation schema accepted: %+v %q", outcome, diagnostics) } } func TestRequirementsPreserveConditionalAndPartialDiscovery(t *testing.T) { nativePromptFixture(t, "http://127.0.0.1:1/v1") unknown := discoverNative(t, "support/summary", "--session-id=session") if unknown.Complete || len(unknown.Issues) == 0 { t.Fatalf("unknown condition claimed complete: %+v", unknown) } for _, input := range unknown.Inputs { if input.Name == "glossary" && input.Kind != promptkit.RequirementConditional { t.Fatalf("unknown branch: %+v", input) } } known := discoverNative(t, "support/summary", "--var=mode=include", "--session-id=session") if !known.Complete { t.Fatalf("known branch incomplete: %+v", known) } for _, input := range known.Inputs { if input.Name == "glossary" && input.Kind != promptkit.RequirementRequired { t.Fatalf("known branch: %+v", input) } } dynamic := discoverNative(t, "support/dynamic") if dynamic.Complete || len(dynamic.Issues) == 0 { t.Fatalf("dynamic name not partial: %+v", dynamic) } resolved := discoverNative(t, "support/dynamic", "--var=source=exact,name") if !resolved.Complete || len(resolved.Inputs) != 1 || resolved.Inputs[0].Name != "exact,name" || resolved.Inputs[0].Declared { t.Fatalf("resolved identifier: %+v", resolved) } unsupported := discoverNative(t, "support/unsupported") var found bool for _, issue := range unsupported.Issues { found = found || issue.Code == promptkit.RequirementIssueUnsupportedConstruct } if unsupported.Complete || !found { t.Fatalf("unsupported analysis not preserved: %+v", unsupported) } } func TestRequirementsSelectionUsesNativeNamespacesAndFailures(t *testing.T) { nativePromptFixture(t, "http://127.0.0.1:1/v1") before := discoverNative(t, "summary", "--prompt-namespace=support", "--var=mode=", "--session-id=session") after := discoverNative(t, "support/summary", "--prompt-namespace=review", "--profile=writer", "--profile-namespace=review", "--var=mode=", "--session-id=session") if after.PromptID != "support/summary" || after.ProfileID != "review/writer" || !reflect.DeepEqual(before.Inputs, after.Inputs) { t.Fatalf("identity/definition changed: before=%+v after=%+v", before, after) } other := discoverNative(t, "review/summary") if other.PromptID != "review/summary" || other.ProfileID != "review/writer" || len(other.Inputs) != 1 || other.Inputs[0].Name != "memo" { t.Fatalf("other definition: %+v", other) } for _, args := range [][]string{ {"promptkit", "requirements", "show", "missing", "--config=app.yml"}, {"promptkit", "requirements", "show", "support/summary", "--config=app.yml", "--profile=missing"}, } { outcome, output, diagnostics := executeNative(t, args...) if outcome.Status == 0 || output != "" || diagnostics == "" { t.Fatalf("selection failure: %+v %q %q", outcome, output, diagnostics) } } } func TestNativeInventoriesInspectionAndPreparationDoNotGenerate(t *testing.T) { var calls atomic.Int64 var fail atomic.Bool server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) w.Header().Set("Content-Type", "application/json") if fail.Load() { w.WriteHeader(http.StatusServiceUnavailable) fmt.Fprint(w, `{"error":{"message":"provider-private-canary"}}`) return } fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"controlled"}}]}`) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") for _, args := range [][]string{ {"promptkit", "list", "prompts", "--all-namespaces"}, {"promptkit", "list", "profiles", "--namespace=support"}, {"promptkit", "list", "backends", "--namespace=support"}, {"promptkit", "list", "namespaces"}, {"promptkit", "list", "collections"}, {"promptkit", "show", "prompt", "support/summary"}, {"promptkit", "show", "profile", "support/writer"}, {"promptkit", "show", "backend", "support/local"}, } { args = append(args, "--config=app.yml", "--format=json") outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 || output == "" { t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) } var value any if err := json.Unmarshal([]byte(output), &value); err != nil { t.Fatal(err) } if args[1] == "show" && args[2] == "prompt" && !strings.Contains(output, "template-canary") { t.Fatalf("native prompt content omitted: %s", output) } } engine, err := loadEngine(context.Background(), ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "app.yml", Set: true}}, applicationInput{}, nil) if err != nil { t.Fatal(err) } nativePrompts, err := engine.ListPrompts(context.Background()) if err != nil { t.Fatal(err) } outcome, output, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--config=app.yml", "--format=json") var listed []promptkit.PromptIdentity if err := json.Unmarshal([]byte(output), &listed); err != nil { t.Fatal(err) } if outcome.Status != 0 || !reflect.DeepEqual(listed, nativePrompts) { t.Fatalf("native default listing changed: %+v %q %q", outcome, output, diagnostics) } outcome, output, diagnostics = executeNative(t, "promptkit", "list", "prompts", "--namespace=review", "--config=app.yml", "--format=json") if outcome.Status != 0 || strings.Contains(output, "support/summary") || !strings.Contains(output, "review/summary") { t.Fatalf("namespace filter lost: %+v %q %q", outcome, output, diagnostics) } writeNativeFile(t, "transcript.md", "artifact-payload-canary") flags := []string{"summary", "--config=app.yml", "--prompt-namespace=support", "--profile=writer", "--profile-namespace=support", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-file=transcript=transcript.md", "--input=unlisted=literal"} for _, command := range [][]string{{"promptkit", "requirements", "validate"}, {"promptkit", "show", "prepared"}} { args := append(append([]string{}, command...), flags...) outcome, output, diagnostics := executeNative(t, append(args, "--format=json")...) if outcome.Status != 0 { t.Fatalf("%v: %+v %q %q", args, outcome, output, diagnostics) } if command[1] == "show" && (!strings.Contains(output, "artifact-payload-canary") || !strings.Contains(output, "support/writer") || !strings.Contains(output, "support/summary")) { t.Fatalf("native prepared content/context lost: %s", output) } } for _, format := range []string{"", "text", "yaml"} { args := append([]string{"promptkit", "show", "prepared"}, flags...) if format != "" { args = append(args, "--format="+format) } outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 || !strings.Contains(output, "artifact-payload-canary") { t.Fatalf("native preview %q: %+v %q %q", format, outcome, output, diagnostics) } } if calls.Load() != 0 { t.Fatal("inspection/preparation contacted provider") } // Smoke tests are the explicit provider-effect boundary. Native findings, // not an application readiness policy, determine what the report says. outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=writer", "--profile-namespace=support", "--format=json") if outcome.Status != 0 || calls.Load() != 1 || !strings.Contains(output, "text_inference") { t.Fatalf("controlled smoke: %+v %q %q calls=%d", outcome, output, diagnostics, calls.Load()) } fail.Store(true) outcome, output, diagnostics = executeNative(t, "promptkit", "test", "text_inference", "--config=app.yml", "--profile=support/writer", "--format=json") var findings struct { Completed bool `json:"completed"` Failure string `json:"failure"` } if err := json.Unmarshal([]byte(output), &findings); err != nil { t.Fatal(err) } if outcome.Status != 0 || !findings.Completed || findings.Failure == "" || strings.Contains(output+diagnostics, "provider-private-canary") { t.Fatalf("native unsuccessful finding changed: %+v %q %q", outcome, output, diagnostics) } writeNativeFile(t, "invalid-app.yml", "server:\n max_request_bytes: -1\n") outcome, output, diagnostics = executeNative(t, "promptkit", "requirements", "show", "support/summary", "--config=invalid-app.yml", "--quiet") if outcome.Kind != commandkit.OutcomeUsage || output != "" || !strings.Contains(diagnostics, "Application configuration") { t.Fatalf("acquisition evidence: %+v %q %q", outcome, output, diagnostics) } }