125 lines
4.3 KiB
Go
125 lines
4.3 KiB
Go
|
|
package cli
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"sync/atomic"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
type auditChangingPartitionClient struct {
|
||
|
|
plans atomic.Int32
|
||
|
|
descriptions atomic.Int32
|
||
|
|
collapse bool
|
||
|
|
}
|
||
|
|
|
||
|
|
func (client *auditChangingPartitionClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
if productionSchemaHasProperty(request, "scenes") {
|
||
|
|
if (client.plans.Add(1) == 1) != client.collapse {
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil
|
||
|
|
}
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"scenes":[{"start_unit_id":1,"end_unit_id":1},{"start_unit_id":2,"end_unit_id":2},{"start_unit_id":3,"end_unit_id":3}]}`}, nil
|
||
|
|
}
|
||
|
|
client.descriptions.Add(1)
|
||
|
|
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Travel","summary":"The party continues its journey."}`}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestSceneCacheRemainsUsableAfterPartitionRenumbering(t *testing.T) {
|
||
|
|
t.Run("expand", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, false) })
|
||
|
|
t.Run("collapse", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, false) })
|
||
|
|
t.Run("expand-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, false, true) })
|
||
|
|
t.Run("collapse-bypass", func(t *testing.T) { auditSceneCacheAfterRepartition(t, true, true) })
|
||
|
|
}
|
||
|
|
|
||
|
|
func auditSceneCacheAfterRepartition(t *testing.T, collapse, bypass bool) {
|
||
|
|
directory := t.TempDir()
|
||
|
|
configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json")
|
||
|
|
configuration := fmt.Sprintf(`version: 6
|
||
|
|
concurrency: {workers: 1}
|
||
|
|
output: {directory: %q}
|
||
|
|
cache_directory: %q
|
||
|
|
promptkit:
|
||
|
|
profiles:
|
||
|
|
- id: audit
|
||
|
|
endpoint: https://example.invalid/v1
|
||
|
|
model: audit
|
||
|
|
api_key_required: false
|
||
|
|
pipelines:
|
||
|
|
audit:
|
||
|
|
llm_profile: audit
|
||
|
|
input: {module: seriatim}
|
||
|
|
chunk:
|
||
|
|
module: dnd/scenes
|
||
|
|
cache: refresh
|
||
|
|
artifacts:
|
||
|
|
scene-descriptions:
|
||
|
|
extract: {module: dnd/scene-descriptions}
|
||
|
|
merge: {module: appendorder}
|
||
|
|
normalize: {module: dnd/scene-descriptions}
|
||
|
|
`, filepath.Join(directory, "output"), filepath.Join(directory, "cache"))
|
||
|
|
if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
transcript := `{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party departs."},{"id":2,"start":1,"end":2,"speaker":"GM","text":"The party travels."},{"id":3,"start":2,"end":3,"speaker":"GM","text":"The party arrives."}]}`
|
||
|
|
if err := os.WriteFile(inputPath, []byte(transcript), 0o600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
client := &auditChangingPartitionClient{collapse: collapse}
|
||
|
|
for run := 1; run <= 2; run++ {
|
||
|
|
var stdout, stderr bytes.Buffer
|
||
|
|
args := []string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--format", "json", "--progress=false"}
|
||
|
|
if bypass {
|
||
|
|
args = append(args, "--cache", "bypass")
|
||
|
|
}
|
||
|
|
status := RunWithOptions(args, &stdout, &stderr, Options{LLMClient: client})
|
||
|
|
if status != 0 {
|
||
|
|
t.Fatalf("run %d: status=%d plans=%d descriptions=%d stderr=%s", run, status, client.plans.Load(), client.descriptions.Load(), stderr.String())
|
||
|
|
}
|
||
|
|
if run == 2 {
|
||
|
|
var report struct {
|
||
|
|
Directory string `json:"output_directory"`
|
||
|
|
}
|
||
|
|
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
content, err := os.ReadFile(filepath.Join(report.Directory, "lanes", "scene-descriptions.json"))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var artifact struct {
|
||
|
|
Scenes []struct {
|
||
|
|
ID string `json:"id"`
|
||
|
|
} `json:"scenes"`
|
||
|
|
}
|
||
|
|
if err := json.Unmarshal(content, &artifact); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
wantCount := 3
|
||
|
|
if collapse {
|
||
|
|
wantCount = 2
|
||
|
|
}
|
||
|
|
if len(artifact.Scenes) != wantCount {
|
||
|
|
t.Fatalf("published scenes: %s", content)
|
||
|
|
}
|
||
|
|
for i, scene := range artifact.Scenes {
|
||
|
|
if scene.ID != fmt.Sprintf("chunk-%06d", i+1) {
|
||
|
|
t.Fatalf("published stale scene identity after repartitioning: %s", content)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
wantCalls := int32(4) // The unchanged final scene is reused across runs.
|
||
|
|
if bypass {
|
||
|
|
wantCalls = 5
|
||
|
|
}
|
||
|
|
if client.descriptions.Load() != wantCalls {
|
||
|
|
t.Fatalf("description calls=%d, want %d", client.descriptions.Load(), wantCalls)
|
||
|
|
}
|
||
|
|
}
|