2026-11-10 17:33:30 +00:00
|
|
|
package cli
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
|
|
|
|
"io"
|
|
|
|
|
"net/http"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"reflect"
|
|
|
|
|
"strings"
|
|
|
|
|
"sync"
|
|
|
|
|
"testing"
|
|
|
|
|
|
2026-11-17 19:57:22 +00:00
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/diagnostics"
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
2026-11-10 17:37:41 +00:00
|
|
|
commandkit "go.maximumdirect.net/commandkit"
|
2026-11-10 17:33:30 +00:00
|
|
|
ck "go.maximumdirect.net/configkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type controlledTransport func(*http.Request) (*http.Response, error)
|
|
|
|
|
|
|
|
|
|
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
|
|
|
|
|
|
|
|
|
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
|
|
|
|
|
var mu sync.Mutex
|
|
|
|
|
var models []string
|
|
|
|
|
var prompts []string
|
2026-11-10 17:37:41 +00:00
|
|
|
proposalCalls := 0
|
2026-11-10 17:33:30 +00:00
|
|
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
|
|
|
|
var request struct {
|
|
|
|
|
Model string `json:"model"`
|
|
|
|
|
Messages []struct {
|
|
|
|
|
Content string `json:"content"`
|
|
|
|
|
} `json:"messages"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
var contents string
|
|
|
|
|
for _, message := range request.Messages {
|
|
|
|
|
contents += message.Content
|
|
|
|
|
}
|
|
|
|
|
mu.Lock()
|
|
|
|
|
models = append(models, request.Model)
|
|
|
|
|
prompts = append(prompts, contents)
|
2026-11-10 17:37:41 +00:00
|
|
|
if request.Model == "model-one" {
|
|
|
|
|
proposalCalls++
|
|
|
|
|
}
|
|
|
|
|
firstProposalCall := proposalCalls
|
2026-11-10 17:33:30 +00:00
|
|
|
mu.Unlock()
|
2026-11-11 17:05:52 +00:00
|
|
|
result := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
2026-11-10 17:33:30 +00:00
|
|
|
switch request.Model {
|
|
|
|
|
case "model-one":
|
2026-11-11 17:05:52 +00:00
|
|
|
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice"}]}`
|
2026-11-10 17:37:41 +00:00
|
|
|
if firstProposalCall == 1 {
|
|
|
|
|
result = `{"invalid":true}`
|
|
|
|
|
}
|
2026-11-10 17:33:30 +00:00
|
|
|
case "model-two":
|
2026-11-11 17:05:52 +00:00
|
|
|
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield"}]}`
|
2026-11-10 17:33:30 +00:00
|
|
|
case "model-three":
|
2026-11-11 17:05:52 +00:00
|
|
|
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast"}]}`
|
2026-11-10 17:33:30 +00:00
|
|
|
}
|
|
|
|
|
encoded, _ := json.Marshal(result)
|
|
|
|
|
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
|
|
|
|
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
|
|
|
|
})
|
|
|
|
|
dir := t.TempDir()
|
|
|
|
|
files := map[string]string{
|
|
|
|
|
"config.yml": `version: 3
|
|
|
|
|
glossary_collections:
|
|
|
|
|
party:
|
|
|
|
|
entries: [{id: alice, name: Alice, category: person}]
|
|
|
|
|
spells:
|
|
|
|
|
entries: [{id: shield, name: Shield, category: spell}]
|
|
|
|
|
protected_terms: [party, spells]
|
|
|
|
|
llm: {validation_profile: review}
|
|
|
|
|
pipeline:
|
|
|
|
|
stages:
|
|
|
|
|
- module: glossary
|
2026-11-10 17:37:41 +00:00
|
|
|
id: "party/../|<tag>"
|
2026-11-10 17:33:30 +00:00
|
|
|
glossaries: [party]
|
|
|
|
|
llm: {proposal_profile: one}
|
|
|
|
|
- module: glossary
|
2026-11-10 17:37:41 +00:00
|
|
|
id: spell
|
2026-11-10 17:33:30 +00:00
|
|
|
glossaries: [spells]
|
|
|
|
|
llm: {proposal_profile: two}
|
|
|
|
|
- module: grammar
|
|
|
|
|
glossaries: [party, spells]
|
|
|
|
|
llm: {proposal_profile: three}
|
|
|
|
|
`,
|
|
|
|
|
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
|
|
|
|
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
|
|
|
|
|
}
|
|
|
|
|
for name, value := range files {
|
|
|
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
2026-11-10 17:37:41 +00:00
|
|
|
loaded.Config.Diagnostics.WorkDir = dir
|
2026-11-10 20:48:32 +00:00
|
|
|
loaded.Config.Reporting.StdoutDir = dir
|
2026-11-10 17:37:41 +00:00
|
|
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
|
2026-11-10 17:33:30 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
defer prepared.Prepared.Output.Close()
|
2026-11-16 15:12:06 +00:00
|
|
|
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
2026-11-10 17:33:30 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
|
|
|
|
|
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
|
|
|
|
|
}
|
2026-11-10 17:37:41 +00:00
|
|
|
if len(prompts) > 4 && !strings.Contains(prompts[4], "Alice casts Sheeld.") {
|
|
|
|
|
t.Fatalf("second proposal missed earlier edit: %s", prompts[4])
|
|
|
|
|
}
|
|
|
|
|
if len(result.Snapshot.WorkUnits) != 3 || result.Snapshot.WorkUnits[0].StageID == nil || *result.Snapshot.WorkUnits[0].StageID != "party/../|<tag>" || result.Snapshot.WorkUnits[1].StageIndex != 1 || result.Snapshot.WorkUnits[2].StageID != nil {
|
|
|
|
|
t.Fatalf("work unit identities: %+v", result.Snapshot.WorkUnits)
|
|
|
|
|
}
|
|
|
|
|
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
|
|
|
|
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
|
|
|
|
}
|
2026-11-16 15:19:59 +00:00
|
|
|
type coordinates struct {
|
|
|
|
|
StageIndex int `json:"stage_index"`
|
|
|
|
|
StageID *string `json:"stage_id"`
|
2026-11-10 17:37:41 +00:00
|
|
|
}
|
2026-11-16 15:19:59 +00:00
|
|
|
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
|
|
|
|
|
check := func(name string, entries []coordinates) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
seen := make(map[int]bool)
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
id, ok := expected[entry.StageIndex]
|
|
|
|
|
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
|
|
|
|
|
t.Fatalf("%s has wrong stage association: %+v", name, entry)
|
|
|
|
|
}
|
|
|
|
|
seen[entry.StageIndex] = true
|
2026-11-10 17:37:41 +00:00
|
|
|
}
|
2026-11-16 15:19:59 +00:00
|
|
|
if len(seen) != len(expected) {
|
|
|
|
|
t.Fatalf("%s missing stage associations: %v", name, seen)
|
2026-11-16 03:37:29 +00:00
|
|
|
}
|
2026-11-16 15:19:59 +00:00
|
|
|
}
|
|
|
|
|
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var ledger []coordinates
|
|
|
|
|
if err := json.Unmarshal(raw, &ledger); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
check("ledger", ledger)
|
|
|
|
|
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var calls struct {
|
|
|
|
|
Calls []struct {
|
|
|
|
|
Context struct {
|
|
|
|
|
coordinates
|
|
|
|
|
Role string
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if err := json.Unmarshal(raw, &calls); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var locations []coordinates
|
|
|
|
|
roles := make(map[int]map[string]bool)
|
|
|
|
|
for _, call := range calls.Calls {
|
|
|
|
|
location := call.Context.coordinates
|
|
|
|
|
locations = append(locations, location)
|
|
|
|
|
if roles[location.StageIndex] == nil {
|
|
|
|
|
roles[location.StageIndex] = make(map[string]bool)
|
|
|
|
|
}
|
|
|
|
|
roles[location.StageIndex][call.Context.Role] = true
|
|
|
|
|
}
|
|
|
|
|
check("LLM observations", locations)
|
|
|
|
|
for index := range expected {
|
|
|
|
|
if !roles[index]["proposal"] || !roles[index]["validation"] {
|
|
|
|
|
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
|
2026-11-10 17:37:41 +00:00
|
|
|
}
|
2026-11-10 17:33:30 +00:00
|
|
|
}
|
2026-11-10 17:37:41 +00:00
|
|
|
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
2026-11-10 17:33:30 +00:00
|
|
|
t.Fatalf("models=%v", models)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
|
|
|
|
dir := t.TempDir()
|
|
|
|
|
files := map[string]string{
|
2026-11-10 20:48:32 +00:00
|
|
|
"config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
|
2026-11-10 17:33:30 +00:00
|
|
|
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
|
|
|
|
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
|
|
|
|
}
|
|
|
|
|
for name, value := range files {
|
|
|
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
|
|
|
|
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
|
|
|
|
|
})
|
|
|
|
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
2026-11-10 20:48:32 +00:00
|
|
|
loaded.Config.Reporting.StdoutDir = dir
|
2026-11-10 17:33:30 +00:00
|
|
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
defer prepared.Prepared.Output.Close()
|
2026-11-16 15:12:06 +00:00
|
|
|
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
2026-11-10 17:33:30 +00:00
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
2026-11-10 17:37:41 +00:00
|
|
|
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" || result.Snapshot.Errors[0].StageID == nil || *result.Snapshot.Errors[0].StageID != "failing" {
|
2026-11-10 17:33:30 +00:00
|
|
|
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
|
|
|
|
|
}
|
|
|
|
|
}
|