112 lines
4.2 KiB
Go
112 lines
4.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/commandkit"
|
|
)
|
|
|
|
func buildNativeExecutable(t *testing.T) string {
|
|
t.Helper()
|
|
binary := filepath.Join(t.TempDir(), "notarius")
|
|
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
|
|
build.Env = append(os.Environ(), "GOWORK=off")
|
|
if output, err := build.CombinedOutput(); err != nil {
|
|
t.Fatalf("build executable: %v\n%s", err, output)
|
|
}
|
|
return binary
|
|
}
|
|
|
|
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
|
binary := buildNativeExecutable(t)
|
|
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
|
|
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
|
|
output, err := help.CombinedOutput()
|
|
if err != nil || !bytes.Contains(output, []byte("pipeline-id")) || !bytes.Contains(output, []byte("--input-file")) {
|
|
t.Fatalf("%v help: %v\n%s", route, err, output)
|
|
}
|
|
missingInput := exec.Command(binary, append(append([]string(nil), route...), "demo")...)
|
|
output, err = missingInput.CombinedOutput()
|
|
var exit *exec.ExitError
|
|
if !errors.As(err, &exit) || exit.ExitCode() != 2 || !bytes.Contains(output, []byte("--input-file")) {
|
|
t.Fatalf("%v missing input: %v\n%s", route, err, output)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
|
|
binary := buildNativeExecutable(t)
|
|
for _, fallback := range []bool{false, true} {
|
|
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
|
if fallback {
|
|
writer.WriteHeader(http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
writer.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
|
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
|
|
root := t.TempDir()
|
|
configuration := fmt.Sprintf(`version: 6
|
|
output: {directory: output}
|
|
pipelines:
|
|
demo:
|
|
input: {module: seriatim}
|
|
chunk: {module: generic}
|
|
llm_profile: local-fake
|
|
mode: permissive
|
|
max_additional_turns: 0
|
|
artifacts:
|
|
spells:
|
|
extract: {module: dnd/spells}
|
|
promptkit:
|
|
profiles:
|
|
- id: local-fake
|
|
endpoint: %q
|
|
model: fake-model
|
|
api_key_required: false
|
|
`, server.URL+"/v1")
|
|
file := filepath.Join(root, "config.yml")
|
|
input := filepath.Join(root, "input.json")
|
|
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
|
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
|
|
var output, diagnostics bytes.Buffer
|
|
command.Stdout, command.Stderr = &output, &diagnostics
|
|
if err := command.Run(); err != nil {
|
|
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
|
|
}
|
|
if format == commandkit.FormatText {
|
|
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
|
|
t.Fatalf("completion text: %s", output.String())
|
|
}
|
|
} else {
|
|
report := decodeCompletionReport(t, format, output.Bytes())
|
|
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
|
|
t.Fatalf("completion report: %#v", report)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|