package cli import ( "bytes" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" "go.maximumdirect.net/commandkit" ) func buildNativeExecutable(t *testing.T) string { t.Helper() binary := filepath.Join(t.TempDir(), "notarius") build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius") build.Env = append(os.Environ(), "GOWORK=off") if output, err := build.CombinedOutput(); err != nil { t.Fatalf("build executable: %v\n%s", err, output) } return binary } func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) { binary := buildNativeExecutable(t) for _, route := range [][]string{{"run"}, {"pipeline", "run"}} { help := exec.Command(binary, append(append([]string(nil), route...), "--help")...) output, err := help.CombinedOutput() if err != nil || !bytes.Contains(output, []byte("pipeline-id")) || !bytes.Contains(output, []byte("--input-file")) { t.Fatalf("%v help: %v\n%s", route, err, output) } missingInput := exec.Command(binary, append(append([]string(nil), route...), "demo")...) output, err = missingInput.CombinedOutput() var exit *exec.ExitError if !errors.As(err, &exit) || exit.ExitCode() != 2 || !bytes.Contains(output, []byte("--input-file")) { t.Fatalf("%v missing input: %v\n%s", route, err, output) } } } func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) { binary := buildNativeExecutable(t) for _, fallback := range []bool{false, true} { server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if fallback { writer.WriteHeader(http.StatusServiceUnavailable) return } writer.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}}) })) t.Cleanup(server.Close) for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} { t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) { root := t.TempDir() configuration := fmt.Sprintf(`version: 6 output: {directory: output} pipelines: demo: input: {module: seriatim} chunk: {module: generic} llm_profile: local-fake mode: permissive max_additional_turns: 0 artifacts: spells: extract: {module: dnd/spells} promptkit: profiles: - id: local-fake endpoint: %q model: fake-model api_key_required: false `, server.URL+"/v1") file := filepath.Join(root, "config.yml") input := filepath.Join(root, "input.json") for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} { if err := os.WriteFile(path, []byte(content), 0600); err != nil { t.Fatal(err) } } command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose") var output, diagnostics bytes.Buffer command.Stdout, command.Stderr = &output, &diagnostics if err := command.Run(); err != nil { t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String()) } if format == commandkit.FormatText { if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") { t.Fatalf("completion text: %s", output.String()) } } else { report := decodeCompletionReport(t, format, output.Bytes()) if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil { t.Fatalf("completion report: %#v", report) } if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil { t.Fatal(err) } } }) } } }