Files
audita/cmd/audita/main_integration_test.go

1642 lines
63 KiB
Go

package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/cli"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gopkg.in/yaml.v3"
)
func TestProcessSubprocessHelper(t *testing.T) {
if os.Getenv("AUDITA_HELPER_PROCESS") != "1" {
return
}
for i, arg := range os.Args {
if arg == "--" {
if value := os.Getenv("AUDITA_TEST_UMASK"); value != "" {
mask, err := strconv.ParseUint(value, 8, 32)
if err != nil {
os.Exit(2)
}
syscall.Umask(int(mask))
}
os.Exit(run(os.Args[i+1:], os.Stdout, os.Stderr))
}
}
os.Exit(2)
}
type fixtureProvider struct {
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
validationRequests [][]int
extraDecisionSent bool
mode string
secret string
entered chan struct{}
beforeResponse func()
}
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
var request struct {
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
ResponseFormat struct {
JSONSchema struct {
Schema struct {
Properties map[string]any `json:"properties"`
} `json:"schema"`
} `json:"json_schema"`
} `json:"response_format"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
http.Error(w, "bad request", 400)
return
}
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
var requestedIndices []int
if validation && strings.HasPrefix(p.mode, "decision-") {
var err error
requestedIndices, err = fixtureValidationIndices(request.Messages)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
p.mu.Lock()
p.calls++
if validation {
p.validationCalls++
} else {
p.proposalCalls++
}
validationCall := p.validationCalls
proposalCall := p.proposalCalls
if validation && strings.HasPrefix(p.mode, "decision-") {
p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...))
}
addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent
if addExtra {
p.extraDecisionSent = true
}
beforeResponse := p.beforeResponse
p.mu.Unlock()
if beforeResponse != nil {
beforeResponse()
}
if p.mode == "wait-cancel" {
select {
case p.entered <- struct{}{}:
default:
}
<-r.Context().Done()
return
}
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(w, `{"error":{"message":"provider unavailable"}}`)
return
}
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
if p.mode == "malformed-response" {
content = `{"corrections": [`
}
if p.mode == "repair-filter-later-failure" && !validation {
if proposalCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X"},{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
}
}
if strings.HasPrefix(p.mode, "span-") {
switch p.mode {
case "span-clean", "span-filter":
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"}]}`
if p.mode == "span-filter" {
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"},{"id":3,"original_text":"uh","corrected_text":""}]}`
}
case "span-spacing":
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" "}]}`
case "span-negative":
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":""}]}`
case "span-ambiguous":
content = `{"corrections":[{"id":1,"original_text":"Mage Armor","corrected_text":"ward"}]}`
if _, repair := request.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; repair {
content = `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`
}
}
if validation {
if p.mode == "span-clean" || p.mode == "span-filter" {
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"},{"correction_index":2,"approved":true,"reason":"ok"}]}`
} else {
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
}
if p.mode == "span-negative" && validationCall == 2 {
content = `{"validations":[{"correction_index":1,"approved":false,"reason":"would reverse meaning"}]}`
}
}
}
if strings.HasPrefix(p.mode, "decision-") && !validation {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"}]}`
if p.mode == "decision-mixed" {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"},{"id":2,"original_text":"cat","corrected_text":"dog"},{"id":3,"original_text":"bad","corrected_text":"good"},{"id":4,"original_text":"slow","corrected_text":"fast"}]}`
}
}
if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
if p.mode == "invalid-decision-after-repair" {
if validationCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"validations":[{"correction_index":9,"approved":true,"reason":"ok"}]}`
}
}
}
if validation && strings.HasPrefix(p.mode, "decision-") {
decisions := make([]map[string]any, 0, len(requestedIndices)+2)
for _, index := range requestedIndices {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
continue
}
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "reason": "conflicting review"})
}
}
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "reason": "unexpected review"})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
}
encoded, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
// Read the supplied candidate JSON independently of the bundled instruction prose.
type fixtureValidationItem struct {
CorrectionIndex int `json:"correction_index"`
OriginalText string `json:"original_text"`
}
func fixtureValidationItems(content string) ([]fixtureValidationItem, bool) {
for offset := 0; offset < len(content); offset++ {
if content[offset] != '[' {
continue
}
var items []fixtureValidationItem
if err := json.Unmarshal([]byte(content[offset:]), &items); err == nil && len(items) > 0 && items[0].CorrectionIndex > 0 {
return items, true
}
}
return nil, false
}
func fixtureValidationIndices(messages []struct {
Content string `json:"content"`
}) ([]int, error) {
for _, message := range messages {
if items, ok := fixtureValidationItems(message.Content); ok {
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices, nil
}
}
return nil, fmt.Errorf("validation request items not found")
}
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
type processFixture struct {
dir string
config string
transcript string
output string
provider *fixtureProvider
server *httptest.Server
}
func newProcessFixture(t *testing.T, mode, modules string) processFixture {
t.Helper()
dir := t.TempDir()
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
configPath := filepath.Join(dir, "config.yml")
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
repairs = 1
}
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
var stageList []string
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
if name = strings.TrimSpace(name); name != "" {
stageList = append(stageList, " - module: "+name)
}
}
pipeline := " stages: []\n"
if len(stageList) != 0 {
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
}
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n correction_repair_attempts: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
t.Fatal(err)
}
transcript := filepath.Join(dir, "input.json")
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
t.Fatal(err)
}
return processFixture{dir: dir, config: configPath, transcript: transcript,
output: filepath.Join(dir, "output.json"), provider: p, server: server}
}
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
t.Helper()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
if includeOutput {
args = append(args, "--output", f.output)
}
args = append(args, extra...)
command := exec.Command(os.Args[0], args...)
command.Dir = f.dir
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stdout, stderr bytes.Buffer
command.Stdout, command.Stderr = &stdout, &stderr
err := command.Run()
code := 0
if err != nil {
if exit, ok := err.(*exec.ExitError); ok {
code = exit.ExitCode()
} else {
t.Fatal(err)
}
}
return code, stdout.String(), stderr.String()
}
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
t.Helper()
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
if err != nil {
t.Fatal(err)
}
var document reporting.ErrorsDocument
if err := yaml.Unmarshal(raw, &document); err != nil {
t.Fatal(err)
}
return document
}
func (f processFixture) recoveries(t *testing.T) reporting.RecoveriesDocument {
t.Helper()
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
if err != nil {
t.Fatal(err)
}
var document reporting.RecoveriesDocument
if err := yaml.Unmarshal(raw, &document); err != nil {
t.Fatal(err)
}
return document
}
func (f processFixture) setSegments(t *testing.T, texts ...string) {
t.Helper()
segments := make([]schema.Segment, len(texts))
for i, value := range texts {
segments[i] = schema.Segment{ID: i + 1, Speaker: "A", Start: float64(i * 10), End: float64(i*10 + 1), Text: value, Categories: []string{"dialogue"}}
}
raw, err := json.Marshal(segments)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(f.transcript, raw, 0600); err != nil {
t.Fatal(err)
}
}
func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transcript {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
transcript, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
return transcript.Canonical()
}
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {
t.Helper()
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != 1 {
t.Fatalf("expected one debug run: %v %v", dirs, err)
}
normalized := f.readTranscript(t, filepath.Join(dirs[0], "normalized-transcript.json"))
raw, err := os.ReadFile(filepath.Join(dirs[0], "correction-ledger.json"))
if err != nil {
t.Fatal(err)
}
var ledger []map[string]json.RawMessage
if err := json.Unmarshal(raw, &ledger); err != nil {
t.Fatal(err)
}
return normalized, ledger
}
func assertSpanMetadata(t *testing.T, normalized, output *schema.Transcript) {
t.Helper()
if len(normalized.Segments) != len(output.Segments) {
t.Fatalf("segment membership changed: normalized=%+v output=%+v", normalized, output)
}
for i := range normalized.Segments {
normalized.Segments[i].Text = output.Segments[i].Text
}
if !reflect.DeepEqual(normalized, output) {
t.Fatalf("segment metadata/order changed: normalized=%+v output=%+v", normalized, output)
}
}
func ledgerDisposition(entry map[string]json.RawMessage) string {
var value string
_ = json.Unmarshal(entry["disposition"], &value)
return value
}
func ledgerHasDecision(t *testing.T, entry map[string]json.RawMessage, field, key string, approved bool) bool {
t.Helper()
var decisions []struct {
ValidatorKey string `json:"validator_key"`
Approved bool `json:"approved"`
}
if err := json.Unmarshal(entry[field], &decisions); err != nil {
t.Fatal(err)
}
for _, decision := range decisions {
if decision.ValidatorKey == key && decision.Approved == approved {
return true
}
}
return false
}
func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
t.Helper()
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != 1 {
t.Fatalf("debug run: %v %v", dirs, err)
}
raw, err := os.ReadFile(filepath.Join(dirs[0], "llm-execution.json"))
if err != nil {
t.Fatal(err)
}
var execution struct {
Calls []struct {
Summary struct {
ProviderAttempts int `json:"provider_attempts"`
StructuralRepairs int `json:"structural_repairs"`
} `json:"summary"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &execution); err != nil {
t.Fatal(err)
}
if len(execution.Calls) != wantCalls || f.provider.callCount() != wantCalls {
t.Fatalf("unexpected generation counts: calls=%d provider=%d", len(execution.Calls), f.provider.callCount())
}
for _, call := range execution.Calls {
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
t.Fatalf("unexpected structural repair: %+v", call)
}
}
}
func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage {
t.Helper()
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json"))
if err != nil {
t.Fatal(err)
}
var diagnostics []map[string]json.RawMessage
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(raw, &payload); err != nil {
t.Fatal(err)
}
if _, validation := payload["reconciliation"]; validation {
diagnostics = append(diagnostics, payload)
}
}
return diagnostics
}
func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int {
t.Helper()
f.provider.mu.Lock()
requests := append([][]int(nil), f.provider.validationRequests...)
providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls
f.provider.mu.Unlock()
if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations {
t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests)
}
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
if err != nil || len(paths) != 1 {
t.Fatalf("execution path: %v %v", paths, err)
}
raw, err := os.ReadFile(paths[0])
if err != nil {
t.Fatal(err)
}
var execution struct {
Calls []struct {
Summary struct {
ProviderAttempts int `json:"provider_attempts"`
StructuralRepairs int `json:"structural_repairs"`
} `json:"summary"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &execution); err != nil {
t.Fatal(err)
}
if len(execution.Calls) != providerCalls {
t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls)
}
for _, call := range execution.Calls {
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
t.Fatalf("extra generation or structural repair: %+v", call)
}
}
metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json"))
if err != nil {
t.Fatal(err)
}
validatorsSeen := make(map[string]bool)
for _, path := range metadataPaths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var metadata struct {
ValidatorName string `json:"validator_name"`
BatchIndex int `json:"batch_index"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
if metadata.ValidatorName == "" {
continue
}
if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 {
t.Fatalf("duplicate or split validation call: %+v", metadata)
}
validatorsSeen[metadata.ValidatorName] = true
}
if len(validatorsSeen) != validations {
t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations)
}
return requests
}
func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
f := newProcessFixture(t, "decision-mixed", "[grammar]")
f.setSegments(t, "teh", "cat", "bad", "slow")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" {
t.Fatalf("partial validation transcript: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries)
}
if len(ledger) != 4 {
t.Fatalf("unexpected candidate ledger entries: %+v", ledger)
}
for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} {
entry := ledger[index]
var candidate int
var reason string
if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 {
t.Fatal(err)
}
if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) {
t.Fatalf("candidate %d evidence: %+v", index, entry)
}
if index > 0 {
var decisions []struct {
Disposition string `json:"disposition"`
}
if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition {
t.Fatalf("candidate %d decision: %+v %v", index, decisions, err)
}
}
}
var mixedSeen bool
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
UnavailableIndices []int `json:"unavailable_indices"`
IgnoredIndices []int `json:"ignored_indices"`
Decisions []struct {
ProposalIndex int `json:"proposal_index"`
ReasonCode string `json:"reason_code"`
} `json:"decisions"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) {
continue
}
var decoded struct {
Validations []struct {
CorrectionIndex int `json:"correction_index"`
} `json:"validations"`
}
if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil {
t.Fatal(err)
}
var finalContent string
if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" {
t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded)
}
mixedSeen = true
}
if !mixedSeen {
t.Fatal("missing mixed decision interaction evidence")
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
full, survivor := 0, 0
for _, indices := range requests {
switch {
case reflect.DeepEqual(indices, []int{1, 2, 3, 4}):
full++
case reflect.DeepEqual(indices, []int{1}):
survivor++
default:
t.Fatalf("unexpected validation identities: %v", requests)
}
}
if full != 1 || survivor == 0 {
t.Fatalf("wrong validation progression: %v", requests)
}
}
func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
f := newProcessFixture(t, "decision-extra-only", "[grammar]")
f.setSegments(t, "teh")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
if len(output.Segments) != 1 || output.Segments[0].Text != "the" {
t.Fatalf("lost usable correction: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries)
}
markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) {
t.Fatalf("Markdown totals disagree: %s", markdown)
}
_, ledger := f.debugArtifacts(t)
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" {
t.Fatalf("unknown candidate in ledger: %+v", ledger)
}
ignored := 0
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
IgnoredIndices []int `json:"ignored_indices"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) {
ignored++
}
}
if ignored != 1 {
t.Fatalf("expected one handled extra decision, got %d", ignored)
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
for _, indices := range requests {
if !reflect.DeepEqual(indices, []int{1}) {
t.Fatalf("unexpected identities: %v", requests)
}
}
}
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
f := newProcessFixture(t, "span-clean", "[spoken_word]")
f.setSegments(t, "uh hello", "I I think")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 2 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" {
t.Fatalf("unexpected corrected transcript: %+v", output)
}
if len(ledger) != 2 || ledgerDisposition(ledger[0]) != "applied" || ledgerDisposition(ledger[1]) != "applied" || string(ledger[0]["proposed_corrected_text"]) != `""` || string(ledger[0]["applied_corrected_text"]) != `""` {
t.Fatalf("deletion missing from ledger: %+v", ledger)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 0 || *errors.Summary.CorrectionsApplied != 2 || *errors.Summary.WorkUnitsCompleted != 1 || *recoveries.Summary.CorrectionsApplied != 2 {
t.Fatalf("report totals disagree: errors=%+v recoveries=%+v", errors, recoveries)
}
f.assertNoStructuralRepairs(t, 3)
}
func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) {
f := newProcessFixture(t, "span-spacing", "[grammar]")
f.setSegments(t, "hello world")
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
output := f.readTranscript(t, f.output)
normalized, _ := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 1 || output.Segments[0].Text != "hello world" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
t.Fatalf("spacing correction failed: %+v", output)
}
f.assertNoStructuralRepairs(t, 3)
}
func TestSubprocessRejectsAmbiguousSpanAtTheSameBoundaryWithOrWithoutRepair(t *testing.T) {
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
f := newProcessFixture(t, "span-ambiguous", "[grammar]")
f.setSegments(t, "Mage Armor and Mage Armor")
raw, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
cfg := strings.Replace(string(raw), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", budget), 1)
cfg += "protected_terms: [spells]\nglossary_collections:\n spells:\n entries:\n - {id: armor, name: Mage Armor, category: spell}\n"
if err := os.WriteFile(f.config, []byte(cfg), 0600); err != nil {
t.Fatal(err)
}
if code, _, stderr := f.run(t, true, "--debug"); code != 0 {
t.Fatalf("exit %d: %s", code, stderr)
}
_, ledger := f.debugArtifacts(t)
if len(ledger) != 1 || !ledgerHasDecision(t, ledger[0], "deterministic_validator_decisions", "original_text_presence", false) ||
ledgerHasDecision(t, ledger[0], "deterministic_validator_decisions", "protected_terms", true) {
t.Fatalf("unexpected evidence: %+v", ledger)
}
if text := f.readTranscript(t, f.output).Segments[0].Text; text != "Mage Armor and Mage Armor" {
t.Errorf("ambiguous proposal changed segment to %q", text)
}
f.assertNoStructuralRepairs(t, 1+budget)
})
}
}
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) {
f := newProcessFixture(t, "span-filter", "[spoken_word]")
f.setSegments(t, "uh hello", "I I think", "uh")
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 3 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" || output.Segments[2].Text != "uh" {
t.Fatalf("filtered segment or survivors wrong: %+v", output)
}
recoveries := f.recoveries(t)
if len(f.errors(t).Errors) != 0 || len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].Reasons) != 1 || recoveries.Recoveries[0].Reasons[0] != "proposals_filtered" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || recoveries.Recoveries[0].FilteredProposals[0].Reason != "empty_resulting_segment" || *recoveries.Summary.ProposalResponsesWithFiltering != 1 {
t.Fatalf("filtering recovery wrong: %+v", recoveries)
}
if len(ledger) != 3 || ledgerDisposition(ledger[2]) != "rejected" || string(ledger[2]["proposed_corrected_text"]) != `""` {
t.Fatalf("filtered deletion evidence missing: %+v", ledger)
}
if !ledgerHasDecision(t, ledger[2], "deterministic_validator_decisions", "non_empty_resulting_segment", false) {
t.Fatalf("renamed validator missing from evidence: %+v", ledger[2])
}
f.assertNoStructuralRepairs(t, 3)
}
func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
f := newProcessFixture(t, "span-negative", "[grammar]")
f.setSegments(t, "not good")
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 0 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 1 || output.Segments[0].Text != "not good" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
t.Fatalf("negative review changed outcome: %+v", output)
}
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "rejected" || string(ledger[0]["proposed_corrected_text"]) != `""` {
t.Fatalf("rejection evidence missing: %+v", ledger)
}
if _, present := ledger[0]["applied_corrected_text"]; present || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "editorial_review", true) || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "meaning_reversal_review", false) {
t.Fatalf("negative decision not recorded: %+v", ledger[0])
}
f.assertNoStructuralRepairs(t, 3)
}
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("output: %s %v", output, err)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || document.Summary.WorkUnitsCompleted == nil || *document.Summary.WorkUnitsCompleted != 1 || len(document.Errors) != 0 {
t.Fatalf("errors document: %+v", document)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
report, err := os.ReadFile(filepath.Join(f.dir, name))
if err != nil {
t.Fatal(err)
}
if bytes.Contains(report, []byte("hello world")) || bytes.Contains(report, []byte("Hello world")) || bytes.Contains(report, []byte(f.provider.secret)) {
t.Fatalf("ordinary report contains payload or secret: %s", name)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
t.Fatalf("debug created by default: %v", err)
}
}
func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
reports := filepath.Join(f.dir, "reports")
if err := os.Mkdir(reports, 0700); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("stdout run wrote transcript file: %v", err)
}
if _, err := os.Stat(filepath.Join(reports, "errors.yml")); err != nil {
t.Fatal(err)
}
}
func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
transcriptDir := filepath.Join(f.dir, "transcripts")
reportDir := filepath.Join(f.dir, "reports")
debugDir := filepath.Join(f.dir, "debug")
for _, dir := range []string{transcriptDir, reportDir} {
if err := os.Mkdir(dir, 0700); err != nil {
t.Fatal(err)
}
}
f.output = filepath.Join(transcriptDir, "corrected.json")
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("transcript: %s %v", output, err)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(reportDir, name)); err != nil {
t.Fatalf("report %s: %v", name, err)
}
if _, err := os.Stat(filepath.Join(transcriptDir, name)); !os.IsNotExist(err) {
t.Fatalf("report %s appeared beside transcript: %v", name, err)
}
}
dirs, err := filepath.Glob(filepath.Join(debugDir, "run-*"))
if err != nil || len(dirs) != 1 {
t.Fatalf("debug runs: %v %v", dirs, err)
}
if _, err := os.Stat(filepath.Join(dirs[0], "correction-ledger.json")); err != nil {
t.Fatalf("debug capture: %v", err)
}
}
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
var expectedOutput, expectedSummary string
for _, selection := range []struct{ reporting, debug bool }{
{false, false}, {false, true}, {true, false}, {true, true},
} {
f := newProcessFixture(t, "normal", "[grammar]")
reportDir := filepath.Join(f.dir, "missing")
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
if selection.reporting {
reportDir = filepath.Join(f.dir, "selected-reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
}
if selection.debug {
extra = append(extra, "--debug")
}
code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
summary := ""
for _, line := range strings.Split(stderr, "\n") {
if strings.HasPrefix(line, "Completed:") {
summary = line
break
}
}
if expectedOutput == "" {
expectedOutput, expectedSummary = string(output), summary
} else if string(output) != expectedOutput || summary != expectedSummary {
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
_, err := os.Stat(filepath.Join(reportDir, name))
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
}
}
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != boolInt(selection.debug) {
t.Fatalf("selection=%+v dirs=%v %v", selection, dirs, err)
}
if selection.debug {
for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} {
if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil {
t.Fatal(err)
}
}
}
}
}
func boolInt(value bool) int {
if value {
return 1
}
return 0
}
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("hello world")) {
t.Fatalf("normalized output: %s %v", output, err)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
t.Fatalf("model failure report: %+v", document)
}
}
func TestSubprocessCapturesMalformedProviderResponse(t *testing.T) {
f := newProcessFixture(t, "malformed-response", "[grammar]")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
}
responseCaptured := false
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if entry.IsDir() || !strings.HasSuffix(entry.Name(), "-response-payload.json") {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
var response struct {
FinalContent string `json:"final_content"`
}
if err := json.Unmarshal(data, &response); err != nil {
return err
}
responseCaptured = responseCaptured || response.FinalContent == `{"corrections": [`
return nil
})
if err != nil || !responseCaptured {
t.Fatalf("missing malformed provider response: %v", err)
}
}
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("unchecked output: %s %v", output, err)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 4 {
t.Fatalf("validation failure report: %+v; calls=%d", document, f.provider.callCount())
}
}
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("output: %s %v", output, err)
}
errors := f.errors(t)
if len(errors.Errors) != 1 || errors.Errors[0].ModuleInstance != "grammar_2" {
t.Fatalf("later failure: %+v", errors)
}
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
if err != nil {
t.Fatal(err)
}
var recoveries reporting.RecoveriesDocument
if err := yaml.Unmarshal(raw, &recoveries); err != nil {
t.Fatal(err)
}
if len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].ModuleInstance != "grammar_1" || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || *recoveries.Summary.Recoveries != 1 {
t.Fatalf("combined recovery: %+v", recoveries)
}
if !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "output_repaired") || !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "proposals_filtered") {
t.Fatalf("recovery reasons: %+v", recoveries.Recoveries[0])
}
}
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || len(document.Errors[0].Failures) != 1 || document.Errors[0].Failures[0].Reason != "validation_decisions_invalid" {
t.Fatalf("invalid decisions report: %+v", document)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("unchecked output: %s %v", output, err)
}
}
func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T) {
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stderr bytes.Buffer
command.Stderr = &stderr
if err := command.Start(); err != nil {
t.Fatal(err)
}
select {
case <-f.provider.entered:
case <-ctx.Done():
t.Fatalf("provider was not reached: %v", ctx.Err())
}
if err := command.Process.Signal(os.Interrupt); err != nil {
t.Fatal(err)
}
err := command.Wait()
var exit *exec.ExitError
if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 1 {
t.Fatalf("interrupt did not reach the failure exit: %v %v stderr=%q", err, ctx.Err(), stderr.String())
}
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
t.Fatalf("cancellation report: %+v", document)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("canceled run published transcript: %v", err)
}
}
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
document := f.errors(t)
if len(document.Errors) != 1 || document.Errors[0].ModuleInstance != "grammar_1" || *document.Summary.WorkUnitsCompleted != 2 {
t.Fatalf("partial outcome: %+v", document)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("later correction: %s %v", output, err)
}
}
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
}
code, _, _ := f.run(t, true)
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failed transcript published: %v", err)
}
}
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
}
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, "failed-reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
code, _, _ := f.run(t, true, "--report-dir", reportDir)
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
t.Fatalf("report created before preflight: %s %v", name, err)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("failed report appeared in default directory: %v", err)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failed transcript published: %v", err)
}
}
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
f.output = filepath.Join(f.dir, "missing", "output.json")
code, _, stderr := f.run(t, true)
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "report directory") {
t.Fatalf("preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("report initialized before preflight: %v", err)
}
}
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
}
if _, err := os.Stat(f.output); err != nil {
t.Fatal(err)
}
if document := f.errors(t); document.Status != reporting.StatusCompleted {
t.Fatalf("report status: %+v", document)
}
}
func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
f.output = filepath.Join(f.dir, "missing", "output.json")
code, _, stderr := f.run(t, true, "--reporting=false")
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
}
func TestSubprocessInfeasibleSectionTargetIsAnInputFailure(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true, "--target-sections", "10")
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "target_sections") {
t.Fatalf("infeasible target: code=%d stdout=%q stderr=%q calls=%d", code, stdout, stderr, f.provider.callCount())
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("unexpected transcript output: %v", err)
}
if document := f.errors(t); document.Status != reporting.StatusFailed {
t.Fatalf("report status: %+v", document)
}
}
func TestSubprocessMissingCredentialIdentifiesRequiredEnvironmentVariable(t *testing.T) {
f := newProcessFixture(t, "success", "[grammar]")
f.provider.secret = ""
resources := filepath.Join(f.dir, "prompts.yml")
body, err := os.ReadFile(resources)
if err != nil {
t.Fatal(err)
}
body = bytes.Replace(body, []byte(" - id: proposal\n"), []byte(" - id: proposal\n api_key_required: true\n api_key_env: AUDITA_TEST_PROVIDER_KEY\n"), 1)
if err := os.WriteFile(resources, body, 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 1 || stdout != "" || f.provider.callCount() != 0 ||
!strings.Contains(stderr, "AUDITA_TEST_PROVIDER_KEY") || !strings.Contains(stderr, `proposal profile "proposal"`) {
t.Fatalf("missing credential: status=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
document := f.errors(t)
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 ||
document.Errors[0].Reason != "setup_failure" || strings.Contains(document.Errors[0].Message, "AUDITA_TEST_PROVIDER_KEY") {
t.Fatalf("expected compact setup failure in ordinary report: %+v", document)
}
}
func TestSubprocessUnknownProfileFailsBeforeModelDispatchAndOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true, "--proposal-profile", "unknown")
if code != 2 || stdout != "" || f.provider.callCount() != 0 ||
!strings.Contains(stderr, "stage 1 (grammar) proposal profile") || !strings.Contains(stderr, "unknown") {
t.Fatalf("readiness: code=%d stdout=%q stderr=%q calls=%d", code, stdout, stderr, f.provider.callCount())
}
for _, path := range []string{f.output, filepath.Join(f.dir, "errors.yml")} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("unexpected output at %s: %v", path, err)
}
}
}
func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) {
for _, mask := range []string{"0077", "0022"} {
t.Run(mask, func(t *testing.T) {
t.Setenv("AUDITA_TEST_UMASK", mask)
f := newProcessFixture(t, "normal", "[]")
paths := []string{f.output, filepath.Join(f.dir, "errors.yml"), filepath.Join(f.dir, "recoveries.yml"), filepath.Join(f.dir, "report.md")}
parsed, _ := strconv.ParseUint(mask, 8, 32)
want := os.FileMode(0644 &^ parsed)
for attempt := 0; attempt < 2; attempt++ {
code, _, stderr := f.run(t, true)
if code != 0 {
t.Fatalf("run: %d %s", code, stderr)
}
for _, path := range paths {
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != want {
t.Errorf("%s mode=%o want=%o", filepath.Base(path), info.Mode().Perm(), want)
}
if err := os.Chmod(path, 0600); err != nil {
t.Fatal(err)
}
}
want = 0600 // Existing restrictive modes survive replacement under either mask.
}
})
}
}
func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
f := newProcessFixture(t, "normal", "[]")
f.output = " "
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(filepath.Join(f.dir, " "))
if err != nil || !bytes.Contains(output, []byte("hello world")) {
t.Fatalf("missing transcript: %s %v", output, err)
}
if f.errors(t).Status != reporting.StatusCompleted {
t.Fatal("reports did not follow file output")
}
}
func TestSubprocessAnnouncesActualOutputPaths(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprint(fail), func(t *testing.T) {
f := newProcessFixture(t, "normal", "[]")
reportDir := filepath.Join(f.dir, "reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
f.output = filepath.Join(reportDir, "out.json")
debugParent := filepath.Join(f.dir, "diagnostics")
if fail {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err)
}
}
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if !fail && (!strings.Contains(stderr, filepath.Join(reportDir, "report.md")) || !strings.Contains(stderr, debugParent)) {
t.Fatalf("missing destination announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err)
}
dirs, err := os.ReadDir(debugParent)
if !fail && (err != nil || len(dirs) != 1) {
t.Fatalf("debug location: %v %v", dirs, err)
}
if !fail {
if _, err := os.Stat(f.output); err != nil {
t.Fatal(err)
}
}
})
}
}
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, baselineSummary := f.run(t, true, "--verbose")
if code != 0 {
t.Fatal(baselineSummary)
}
baseline, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
f.provider.mu.Lock()
f.provider.beforeResponse = func() {
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != 1 {
t.Errorf("debug directory: %v %v", dirs, err)
return
}
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
t.Error(err)
}
}
f.provider.mu.Unlock()
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
t.Fatalf("changed outcome: %d %q", code, stderr)
}
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
t.Fatalf("missing debug failures: %q", stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Equal(output, baseline) {
t.Fatalf("changed transcript: %s %v", output, err)
}
document := f.errors(t)
if len(document.Errors) != 0 || *document.Summary.Recoveries != 0 {
t.Fatalf("debug failure classified as model incident: %+v", document)
}
}
func TestSubprocessInPlaceTranscript(t *testing.T) {
for _, invalid := range []bool{false, true} {
t.Run(fmt.Sprint(invalid), func(t *testing.T) {
f := newProcessFixture(t, "success", "[grammar]")
f.output = f.transcript
if invalid {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err)
}
}
before, err := os.ReadFile(f.transcript)
if err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
after, err := os.ReadFile(f.transcript)
if err != nil {
t.Fatal(err)
}
if invalid {
if code != 2 || !bytes.Equal(before, after) {
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
}
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after)
}
})
}
}
func TestSubprocessProfileSources(t *testing.T) {
f := newProcessFixture(t, "success", "[]")
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
t.Fatal(err)
}
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
code, _, stderr := f.run(t, true, "--reporting=false")
if code != 0 {
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
}
// Remove the transcript from the profile directory before the next load.
if err := os.Remove(f.output); err != nil {
t.Fatal(err)
}
code, _, stderr = f.run(t, true)
after, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
}
parentDir := filepath.Join(f.dir, "prompts", "catalog")
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "report.md")
for path, content := range map[string]string{
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
child: "prompts:\n - id: child\n extends: {id: source}\n",
resource: "Captured operator message.\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
resourceBefore, err := os.ReadFile(resource)
if err != nil {
t.Fatal(err)
}
f.output = resource
code, _, stderr = f.run(t, true, "--reporting=false")
resourceAfter, err := os.ReadFile(resource)
if err != nil {
t.Fatal(err)
}
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
}
f.output = filepath.Join(f.dir, "output.json")
code, _, stderr = f.run(t, true, "--report-dir", parentDir)
resourceAfter, err = os.ReadFile(resource)
if err != nil {
t.Fatal(err)
}
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
}
}
func TestSubprocessProgressReportingModes(t *testing.T) {
for _, tc := range []struct {
name string
flags []string
visible bool
file bool
}{
{name: "ordinary"},
{name: "explicit", flags: []string{"--progress"}, visible: true},
{name: "debug", flags: []string{"--debug"}, visible: true},
{name: "debug with progress off", flags: []string{"--debug", "--progress=false"}},
{name: "quiet", flags: []string{"--quiet", "--progress"}, file: true},
} {
t.Run(tc.name, func(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, tc.file, tc.flags...)
if code != 0 {
t.Fatalf("exit %d: %s", code, stderr)
}
for _, text := range []string{"Stage 1: grammar", "chunk 1", "proposal", "validation", "provider attempts"} {
if strings.Contains(stderr, text) != tc.visible {
t.Fatalf("progress fragment %q, expected visibility %v: %s", text, tc.visible, stderr)
}
}
if tc.file {
if stdout != "" {
t.Fatalf("unexpected stdout with file output: %q", stdout)
}
} else if !json.Valid([]byte(stdout)) || !strings.Contains(stdout, "Hello world") {
t.Fatalf("stdout is not the transcript: %q", stdout)
}
})
}
}
func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) {
f := newProcessFixture(t, "wait-cancel", "[grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--",
"process", f.transcript, "--config", f.config, "--output", f.output, "--progress")
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
stderr, err := cmd.StderrPipe()
if err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() {
stop()
_ = cmd.Wait()
}()
progress := make(chan string, 1)
go func() {
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "proposal") && strings.Contains(line, "chunk 1") {
select {
case progress <- line:
default:
}
}
}
}()
select {
case <-f.provider.entered:
case <-ctx.Done():
t.Fatal("provider was not reached")
}
select {
case line := <-progress:
if !strings.Contains(line, "grammar") || !strings.Contains(line, "started") {
t.Fatalf("progress omitted the running operation: %q", line)
}
case <-ctx.Done():
t.Fatal("proposal progress did not reach stderr while the provider was blocked")
}
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
err = cmd.Wait()
var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != 1 || ctx.Err() != nil {
t.Fatalf("termination did not reach the failure exit: %v", err)
}
}
type unavailableDiagnostics struct{}
func (unavailableDiagnostics) Write([]byte) (int, error) {
return 0, errors.New("progress destination failed")
}
func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
code := cli.Run(context.Background(), []string{"process", f.transcript, "--config", f.config, "--output", f.output, "--progress"}, io.Discard, unavailableDiagnostics{})
if code != 1 {
t.Fatalf("expected CommandKit presentation failure, got %d", code)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("transcript did not finish: %s, %v", output, err)
}
if doc := f.errors(t); doc.Status != reporting.StatusCompleted {
t.Fatalf("presentation changed report status: %+v", doc)
}
}
type delayedProposalProgress struct {
delay time.Duration
delayed bool
}
func (w *delayedProposalProgress) Write(p []byte) (int, error) {
if !w.delayed && bytes.Contains(p, []byte("proposal started")) {
w.delayed = true
time.Sleep(w.delay)
}
return len(p), nil
}
func TestProgressDeliveryDoesNotSpendModelDeadline(t *testing.T) {
const timeout = 500 * time.Millisecond
for _, showProgress := range []bool{false, true} {
f := newProcessFixture(t, "normal", "[grammar]")
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
config, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
config = bytes.Replace(config, []byte("llm:\n"), []byte("llm:\n operation_timeout: "+timeout.String()+"\n"), 1)
if err := os.WriteFile(f.config, config, 0600); err != nil {
t.Fatal(err)
}
args := []string{"process", f.transcript, "--config", f.config, "--output", f.output}
diagnostics := &delayedProposalProgress{delay: timeout + 250*time.Millisecond}
if showProgress {
args = append(args, "--progress")
}
code := cli.Run(context.Background(), args, io.Discard, diagnostics)
output, err := os.ReadFile(f.output)
if code != 0 || err != nil || !bytes.Contains(output, []byte("Hello world")) || f.provider.callCount() != 3 {
t.Fatalf("progress=%v, exit=%d, requests=%d, output=%q, read error=%v", showProgress, code, f.provider.callCount(), output, err)
}
if diagnostics.delayed != showProgress {
t.Fatalf("progress=%v, delayed=%v", showProgress, diagnostics.delayed)
}
}
}