Files
notarius/internal/cli/native_process_test.go

112 lines
4.2 KiB
Go
Raw Normal View History

package cli
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"go.maximumdirect.net/commandkit"
)
func buildNativeExecutable(t *testing.T) string {
t.Helper()
binary := filepath.Join(t.TempDir(), "notarius")
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
build.Env = append(os.Environ(), "GOWORK=off")
if output, err := build.CombinedOutput(); err != nil {
t.Fatalf("build executable: %v\n%s", err, output)
}
return binary
}
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
binary := buildNativeExecutable(t)
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
output, err := help.CombinedOutput()
if err != nil || !bytes.Contains(output, []byte("pipeline-id")) || !bytes.Contains(output, []byte("--input-file")) {
t.Fatalf("%v help: %v\n%s", route, err, output)
}
missingInput := exec.Command(binary, append(append([]string(nil), route...), "demo")...)
output, err = missingInput.CombinedOutput()
var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != 2 || !bytes.Contains(output, []byte("--input-file")) {
t.Fatalf("%v missing input: %v\n%s", route, err, output)
}
}
}
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
binary := buildNativeExecutable(t)
for _, fallback := range []bool{false, true} {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if fallback {
writer.WriteHeader(http.StatusServiceUnavailable)
return
}
writer.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
}))
t.Cleanup(server.Close)
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
root := t.TempDir()
configuration := fmt.Sprintf(`version: 6
output: {directory: output}
pipelines:
demo:
input: {module: seriatim}
chunk: {module: generic}
llm_profile: local-fake
mode: permissive
max_additional_turns: 0
artifacts:
spells:
extract: {module: dnd/spells}
promptkit:
profiles:
- id: local-fake
endpoint: %q
model: fake-model
api_key_required: false
`, server.URL+"/v1")
file := filepath.Join(root, "config.yml")
input := filepath.Join(root, "input.json")
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
var output, diagnostics bytes.Buffer
command.Stdout, command.Stderr = &output, &diagnostics
if err := command.Run(); err != nil {
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
}
if format == commandkit.FormatText {
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
t.Fatalf("completion text: %s", output.String())
}
} else {
report := decodeCompletionReport(t, format, output.Bytes())
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
t.Fatalf("completion report: %#v", report)
}
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
t.Fatal(err)
}
}
})
}
}
}