package cli import ( "bytes" "context" "encoding/json" "io" "net/http" "os" "path/filepath" "strings" "testing" "go.maximumdirect.net/ckconfig" "go.maximumdirect.net/ckprompt" commandkit "go.maximumdirect.net/commandkit" pk "go.maximumdirect.net/promptkit" ) func TestNativeConfigurationAndResourceCommands(t *testing.T) { dir := t.TempDir() file := filepath.Join(dir, "config.yml") if err := os.WriteFile(file, []byte("version: 3\ncontext:\n description: example\n"), 0644); err != nil { t.Fatal(err) } app, err := buildApplication() if err != nil { t.Fatal(err) } for _, tc := range []struct { args []string contains string }{ {[]string{"--help"}, "audita"}, {[]string{"version"}, "audita"}, {[]string{"process", "--help"}, "transcript"}, {[]string{"config", "show", "--scope", "app", "--config", file}, "example"}, {[]string{"promptkit", "list", "prompts", "--config", file}, "audita/modules.grammar.proposal"}, } { var out, stderr bytes.Buffer result := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: &out, Err: &stderr}) if result.Status != 0 || !strings.Contains(out.String(), tc.contains) { t.Errorf("%v: status=%d out=%q err=%q", tc.args, result.Status, out.String(), stderr.String()) } } } func TestNativeConfigurationSelectionsAndComparison(t *testing.T) { dir := t.TempDir() first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml") for name, data := range map[string]string{first: "context:\n description: before\n", second: "context:\n description: after\n"} { if err := os.WriteFile(name, []byte(data), 0644); err != nil { t.Fatal(err) } } app, err := buildApplication() if err != nil { t.Fatal(err) } var out, stderr bytes.Buffer outcome := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--scope=app", "--config", first, "--against-config", second, "--set=/context/description=\"left\"", "--against-set=/context/description=\"right\""}, commandkit.Streams{Out: &out, Err: &stderr}) if outcome.Status != 3 || !strings.Contains(out.String(), "left") || !strings.Contains(out.String(), "right") { t.Fatalf("status=%d output=%q err=%q", outcome.Status, out.String(), stderr.String()) } out.Reset() stderr.Reset() outcome = app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config", filepath.Join(dir, "missing.yml")}, commandkit.Streams{Out: &out, Err: &stderr}) if outcome.Status == 0 { t.Fatalf("missing file accepted: %s", out.String()) } } func TestExplicitEmptyConfigurationPathFails(t *testing.T) { app, err := buildApplication() if err != nil { t.Fatal(err) } var out, stderr bytes.Buffer result := app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config="}, commandkit.Streams{Out: &out, Err: &stderr}) if result.Status == 0 { t.Fatal("explicit empty path accepted") } } func TestPromptConfigurationComparisonAcquiresEqualSelectionsIndependently(t *testing.T) { dir := t.TempDir() main := filepath.Join(dir, "config.yml") if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } resource := filepath.Join(dir, "prompts.yml") writeProfile := func(model string) { t.Helper() if err := os.WriteFile(resource, []byte("profiles: [{id: local, model: "+model+"}]\n"), 0600); err != nil { t.Fatal(err) } } writeProfile("before") scope := ckprompt.ConfigScope(func(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) { engine, err := loadPromptEngine(ctx, selection, input, state) writeProfile("after") return engine, err }) side := ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{ Config: commandkit.Supplied[string]{Value: main, Set: true}, }} result, err := scope.Compare(context.Background(), ckconfig.ComparisonRequest[struct{}]{Left: side, Right: side}, &invocationState{}) if err != nil { t.Fatal(err) } for _, difference := range result.Value { if strings.Contains(difference.Left.YAML, "before") && strings.Contains(difference.Right.YAML, "after") { return } } t.Fatal("comparison did not capture the model before and after the resource changed") } func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) { dir := t.TempDir() file := filepath.Join(dir, "config.yml") if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil { t.Fatal(err) } resources := `backends: - id: local endpoint: https://example.invalid/v1 smoke_tests: default_profile: fixture authentication: kind: none profiles: - id: fixture backend: local model: sample collections: audita: prompts: - id: modules.grammar.proposal messages: - role: user content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}' output: format: text validation_mode: none ` if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil { t.Fatal(err) } calls := 0 client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { calls++ body := `{}` if req.Method == http.MethodPost { requestBody, err := io.ReadAll(req.Body) if err != nil { t.Fatal(err) } content := "PROMPTKIT_OK" if strings.Contains(string(requestBody), "response_format") { content = `{"status":"ok"}` } encoded, err := json.Marshal(content) if err != nil { t.Fatal(err) } body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}` } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil })} app, err := buildApplicationWithClient(client) if err != nil { t.Fatal(err) } execute := func(args ...string) (commandkit.Outcome, string, string) { t.Helper() var out, stderr bytes.Buffer result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr}) return result, out.String(), stderr.String() } common := []string{"--config", file, "--llm-profile", "fixture"} args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...) args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction") outcome, out, stderr := execute(args...) if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") { t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr) } args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...) args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=") outcome, out, stderr = execute(args...) if outcome.Status != 0 { t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr) } validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...) validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=") outcome, out, stderr = execute(validatorPreview...) if outcome.Status != 0 || !strings.Contains(out, "teh cat") { t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr) } if calls != 0 { t.Fatalf("inspection made %d provider calls", calls) } outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local") if outcome.Status != 0 || calls == 0 { t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr) } previous := calls outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json") if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") { t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr) } } type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) { request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}} result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"}) if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" { t.Fatalf("variables=%v", result.Vars) } if _, changed := request.Vars["ProposalContract"]; changed { t.Fatal("input variables mutated") } generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}} result = applicationPromptRequest(generic, nil) if len(result.Vars) != 1 || result.Vars["x"] != "y" { t.Fatalf("generic request changed: %v", result.Vars) } }