70 lines
2.2 KiB
Go
70 lines
2.2 KiB
Go
|
|
package cli
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
type longProviderFailure struct{}
|
||
|
|
|
||
|
|
func (longProviderFailure) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
return nil, fmt.Errorf("provider failed: %s", strings.Repeat("a detailed upstream failure; ", 250))
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestPermissiveFallbackSurvivesLongNativeCause(t *testing.T) {
|
||
|
|
directory := t.TempDir()
|
||
|
|
configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json")
|
||
|
|
configuration := fmt.Sprintf(`version: 6
|
||
|
|
output: {directory: %q}
|
||
|
|
cache_directory: %q
|
||
|
|
promptkit:
|
||
|
|
profiles:
|
||
|
|
- id: audit
|
||
|
|
endpoint: https://example.invalid/v1
|
||
|
|
model: audit
|
||
|
|
api_key_required: false
|
||
|
|
pipelines:
|
||
|
|
audit:
|
||
|
|
llm_profile: audit
|
||
|
|
mode: permissive
|
||
|
|
max_additional_turns: 0
|
||
|
|
input: {module: seriatim}
|
||
|
|
artifacts:
|
||
|
|
spells:
|
||
|
|
extract: {module: dnd/spells}
|
||
|
|
normalize: {module: dnd/spells}
|
||
|
|
`, filepath.Join(directory, "output"), filepath.Join(directory, "cache"))
|
||
|
|
if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if err := os.WriteFile(inputPath, []byte(`{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`), 0o600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var stdout, stderr bytes.Buffer
|
||
|
|
status := RunWithOptions([]string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--cache", "bypass", "--llm-transient-retries=0", "--format", "json", "--progress=false"}, &stdout, &stderr, Options{LLMClient: longProviderFailure{}})
|
||
|
|
if status != 0 {
|
||
|
|
t.Fatalf("supported empty fallback failed: status=%d stderr=%s", status, stderr.String())
|
||
|
|
}
|
||
|
|
var report struct {
|
||
|
|
FallbackCount int `json:"fallback_count"`
|
||
|
|
Directory string `json:"output_directory"`
|
||
|
|
}
|
||
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if report.FallbackCount != 1 {
|
||
|
|
t.Fatalf("fallback count = %d", report.FallbackCount)
|
||
|
|
}
|
||
|
|
if _, err := os.Stat(filepath.Join(report.Directory, "lanes", "spells.json")); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|