2026-09-24 02:40:48 +00:00
package main
import (
"bytes"
2026-09-24 17:27:11 +00:00
"context"
2026-09-24 02:40:48 +00:00
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
2026-09-24 18:03:25 +00:00
"os/signal"
2026-09-24 02:40:48 +00:00
"path/filepath"
2026-09-25 02:56:44 +00:00
"reflect"
2026-09-24 21:48:18 +00:00
"strconv"
2026-09-24 02:40:48 +00:00
"strings"
"sync"
2026-09-24 18:03:25 +00:00
"syscall"
2026-09-24 02:40:48 +00:00
"testing"
2026-09-24 18:03:25 +00:00
"time"
2026-09-24 02:40:48 +00:00
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
2026-09-25 02:56:44 +00:00
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
2026-09-24 17:48:52 +00:00
"gopkg.in/yaml.v3"
2026-09-24 02:40:48 +00:00
)
func TestProcessSubprocessHelper ( t * testing . T ) {
if os . Getenv ( "AUDITA_HELPER_PROCESS" ) != "1" {
return
}
for i , arg := range os . Args {
if arg == "--" {
2026-09-24 21:48:18 +00:00
if value := os . Getenv ( "AUDITA_TEST_UMASK" ) ; value != "" {
mask , err := strconv . ParseUint ( value , 8 , 32 )
if err != nil {
os . Exit ( 2 )
}
syscall . Umask ( int ( mask ) )
}
2026-09-24 18:03:25 +00:00
ctx , stop := signal . NotifyContext ( context . Background ( ) , os . Interrupt , syscall . SIGTERM )
code := cli . Run ( ctx , os . Args [ i + 1 : ] , os . Stdout , os . Stderr )
stop ( )
os . Exit ( code )
2026-09-24 02:40:48 +00:00
}
}
os . Exit ( 2 )
}
2026-09-24 17:48:52 +00:00
type fixtureProvider struct {
2026-09-25 11:34:26 +00:00
mu sync . Mutex
calls int
proposalCalls int
validationCalls int
validationRequests [ ] [ ] int
extraDecisionSent bool
mode string
secret string
entered chan struct { }
beforeResponse func ( )
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
func ( p * fixtureProvider ) serve ( w http . ResponseWriter , r * http . Request ) {
2026-09-24 02:40:48 +00:00
var request struct {
2026-09-25 11:34:26 +00:00
Messages [ ] struct {
Content string ` json:"content" `
} ` json:"messages" `
2026-09-24 02:40:48 +00:00
ResponseFormat struct {
JSONSchema struct {
Schema struct {
Properties map [ string ] any ` json:"properties" `
} ` json:"schema" `
} ` json:"json_schema" `
} ` json:"response_format" `
}
if err := json . NewDecoder ( r . Body ) . Decode ( & request ) ; err != nil {
2026-09-24 17:48:52 +00:00
http . Error ( w , "bad request" , 400 )
2026-09-24 02:40:48 +00:00
return
}
_ , validation := request . ResponseFormat . JSONSchema . Schema . Properties [ "validations" ]
2026-09-25 11:34:26 +00:00
var requestedIndices [ ] int
if validation && strings . HasPrefix ( p . mode , "decision-" ) {
var err error
requestedIndices , err = fixtureValidationIndices ( request . Messages )
if err != nil {
http . Error ( w , err . Error ( ) , http . StatusBadRequest )
return
}
}
2026-09-24 02:40:48 +00:00
p . mu . Lock ( )
2026-09-24 17:48:52 +00:00
p . calls ++
if validation {
p . validationCalls ++
2026-09-24 18:03:25 +00:00
} else {
p . proposalCalls ++
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
validationCall := p . validationCalls
2026-09-24 18:03:25 +00:00
proposalCall := p . proposalCalls
2026-09-25 11:34:26 +00:00
if validation && strings . HasPrefix ( p . mode , "decision-" ) {
p . validationRequests = append ( p . validationRequests , append ( [ ] int ( nil ) , requestedIndices ... ) )
}
addExtra := p . mode == "decision-extra-only" && validation && ! p . extraDecisionSent
if addExtra {
p . extraDecisionSent = true
}
2026-09-24 21:48:18 +00:00
beforeResponse := p . beforeResponse
2026-09-24 02:40:48 +00:00
p . mu . Unlock ( )
2026-09-24 21:48:18 +00:00
if beforeResponse != nil {
beforeResponse ( )
}
2026-09-24 18:03:25 +00:00
if p . mode == "wait-cancel" {
select {
case p . entered <- struct { } { } :
default :
}
<- r . Context ( ) . Done ( )
return
}
if p . mode == "all-model-failure" || ( p . mode == "all-validation-failure" && validation ) || ( p . mode == "partial-validation" && validation && validationCall == 1 ) || ( p . mode == "repair-filter-later-failure" && ! validation && proposalCall == 3 ) {
2026-09-24 02:40:48 +00:00
w . WriteHeader ( http . StatusInternalServerError )
_ , _ = fmt . Fprintf ( w , ` { "error": { "message":"provider echoed %s"}} ` , p . secret )
return
}
2026-09-24 17:48:52 +00:00
content := ` { "corrections":[ { "id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]} `
2026-09-24 18:03:25 +00:00
if p . mode == "repair-filter-later-failure" && ! validation {
if proposalCall == 1 {
content = ` { "wrong":true} `
} else {
content = ` { "corrections":[ { "id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99}, { "id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]} `
}
}
2026-09-25 02:56:44 +00:00
if strings . HasPrefix ( p . mode , "span-" ) {
switch p . mode {
case "span-clean" , "span-filter" :
content = ` { "corrections":[ { "id":1,"original_text":"uh ","corrected_text":"","confidence":0.99}, { "id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]} `
if p . mode == "span-filter" {
content = ` { "corrections":[ { "id":1,"original_text":"uh ","corrected_text":"","confidence":0.99}, { "id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}, { "id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]} `
}
case "span-spacing" :
content = ` { "corrections":[ { "id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]} `
case "span-negative" :
content = ` { "corrections":[ { "id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]} `
}
if validation {
if p . mode == "span-clean" || p . mode == "span-filter" {
2026-09-25 12:38:23 +00:00
content = ` { "validations":[ { "correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}, { "correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]} `
2026-09-25 02:56:44 +00:00
} else {
2026-09-25 12:38:23 +00:00
content = ` { "validations":[ { "correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]} `
2026-09-25 02:56:44 +00:00
}
if p . mode == "span-negative" && validationCall == 2 {
2026-09-25 12:38:23 +00:00
content = ` { "validations":[ { "correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]} `
2026-09-25 02:56:44 +00:00
}
}
}
2026-09-25 11:34:26 +00:00
if strings . HasPrefix ( p . mode , "decision-" ) && ! validation {
content = ` { "corrections":[ { "id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]} `
if p . mode == "decision-mixed" {
content = ` { "corrections":[ { "id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}, { "id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99}, { "id":3,"original_text":"bad","corrected_text":"good","confidence":0.99}, { "id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]} `
}
}
2026-09-25 02:56:44 +00:00
if validation && ! strings . HasPrefix ( p . mode , "span-" ) {
2026-09-25 12:38:23 +00:00
content = ` { "validations":[ { "correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]} `
if p . mode == "invalid-decision-after-repair" {
2026-09-24 18:03:25 +00:00
if validationCall == 1 {
content = ` { "wrong":true} `
} else {
content = ` { "validations":[ { "correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]} `
}
}
2026-09-24 02:40:48 +00:00
}
2026-09-25 11:34:26 +00:00
if validation && strings . HasPrefix ( p . mode , "decision-" ) {
decisions := make ( [ ] map [ string ] any , 0 , len ( requestedIndices ) + 2 )
for _ , index := range requestedIndices {
2026-09-25 12:38:23 +00:00
if p . mode == "decision-mixed" && len ( requestedIndices ) == 4 && index == 3 {
2026-09-25 11:34:26 +00:00
continue
}
2026-09-25 12:38:23 +00:00
approved := ! ( p . mode == "decision-mixed" && len ( requestedIndices ) == 4 && index == 2 )
2026-09-25 11:34:26 +00:00
decisions = append ( decisions , map [ string ] any { "correction_index" : index , "approved" : approved , "confidence" : 0.99 , "reason" : "controlled review" } )
2026-09-25 12:38:23 +00:00
if p . mode == "decision-mixed" && len ( requestedIndices ) == 4 && index == 4 {
2026-09-25 11:34:26 +00:00
decisions = append ( decisions , map [ string ] any { "correction_index" : index , "approved" : false , "confidence" : 0.99 , "reason" : "conflicting review" } )
}
}
if p . mode == "decision-mixed" && len ( requestedIndices ) == 4 || addExtra {
decisions = append ( decisions , map [ string ] any { "correction_index" : 99 , "approved" : true , "confidence" : 0.99 , "reason" : "unexpected review" } )
}
raw , _ := json . Marshal ( map [ string ] any { "validations" : decisions } )
content = string ( raw )
}
2026-09-24 17:48:52 +00:00
encoded , _ := json . Marshal ( content )
2026-09-24 02:40:48 +00:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2026-09-24 17:48:52 +00:00
_ , _ = fmt . Fprintf ( w , ` { "model":"synthetic","choices":[ { "message": { "content":%s}}],"usage": { "prompt_tokens":2,"completion_tokens":3,"total_tokens":5}} ` , encoded )
2026-09-24 02:40:48 +00:00
}
2026-09-25 11:34:26 +00:00
func fixtureValidationIndices ( messages [ ] struct {
Content string ` json:"content" `
} ) ( [ ] int , error ) {
for _ , message := range messages {
const marker = "Corrections to validate:"
position := strings . LastIndex ( message . Content , marker )
if position < 0 {
continue
}
var items [ ] struct {
CorrectionIndex int ` json:"correction_index" `
}
if err := json . Unmarshal ( [ ] byte ( strings . TrimSpace ( message . Content [ position + len ( marker ) : ] ) ) , & items ) ; err != nil {
return nil , fmt . Errorf ( "decode validation request items: %w" , err )
}
indices := make ( [ ] int , len ( items ) )
for i , item := range items {
indices [ i ] = item . CorrectionIndex
}
return indices , nil
}
return nil , fmt . Errorf ( "validation request items not found" )
}
2026-09-24 17:48:52 +00:00
func ( p * fixtureProvider ) callCount ( ) int { p . mu . Lock ( ) ; defer p . mu . Unlock ( ) ; return p . calls }
2026-09-24 02:40:48 +00:00
type processFixture struct {
2026-09-24 17:48:52 +00:00
dir string
config string
transcript string
glossary string
output string
provider * fixtureProvider
server * httptest . Server
2026-09-24 02:40:48 +00:00
}
func newProcessFixture ( t * testing . T , mode , modules string ) processFixture {
t . Helper ( )
dir := t . TempDir ( )
2026-09-24 18:03:25 +00:00
p := & fixtureProvider { mode : mode , secret : ` selected-secret-"\-123 ` , entered : make ( chan struct { } , 1 ) }
2026-09-24 02:40:48 +00:00
server := httptest . NewServer ( http . HandlerFunc ( p . serve ) )
t . Cleanup ( server . Close )
2026-09-24 17:48:52 +00:00
if err := os . Mkdir ( filepath . Join ( dir , "profiles" ) , 0700 ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
for _ , id := range [ ] string { "proposal" , "validation" } {
profile := fmt . Sprintf ( "id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n" , id )
if err := os . WriteFile ( filepath . Join ( dir , "profiles" , id + ".yaml" ) , [ ] byte ( profile ) , 0600 ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
}
configPath := filepath . Join ( dir , "config.yml" )
2026-09-24 18:03:25 +00:00
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
repairs = 1
}
2026-09-25 11:34:26 +00:00
if strings . HasPrefix ( mode , "span-" ) || strings . HasPrefix ( mode , "decision-" ) {
2026-09-25 02:56:44 +00:00
repairs = 1
}
2026-09-24 18:03:25 +00:00
configText := fmt . Sprintf ( "version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n" , server . URL , repairs , modules , filepath . Join ( dir , "runs" ) )
2026-09-25 11:34:26 +00:00
if strings . HasPrefix ( mode , "span-" ) || strings . HasPrefix ( mode , "decision-" ) {
2026-09-25 02:56:44 +00:00
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
2026-09-24 02:40:48 +00:00
if err := os . WriteFile ( configPath , [ ] byte ( configText ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
transcript := filepath . Join ( dir , "input.json" )
if err := os . WriteFile ( transcript , [ ] byte ( ` [ { "id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}] ` ) , 0600 ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
glossary := filepath . Join ( dir , "glossary.yaml" )
if err := os . WriteFile ( glossary , [ ] byte ( "glossary:\n - name: Example\n category: product\n summary: A sample.\n" ) , 0600 ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
return processFixture { dir : dir , config : configPath , transcript : transcript , glossary : glossary ,
output : filepath . Join ( dir , "output.json" ) , provider : p , server : server }
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
func ( f processFixture ) run ( t * testing . T , includeOutput bool , extra ... string ) ( int , string , string ) {
2026-09-24 02:40:48 +00:00
t . Helper ( )
2026-09-24 17:48:52 +00:00
args := [ ] string { "-test.run=^TestProcessSubprocessHelper$" , "--" , "process" , f . transcript , "--config" , f . config , "--glossary" , f . glossary }
if includeOutput {
args = append ( args , "--output" , f . output )
}
2026-09-24 02:40:48 +00:00
args = append ( args , extra ... )
2026-09-24 17:48:52 +00:00
command := exec . Command ( os . Args [ 0 ] , args ... )
2026-09-24 21:48:18 +00:00
command . Dir = f . dir
2026-09-24 17:48:52 +00:00
command . Env = append ( os . Environ ( ) , "AUDITA_HELPER_PROCESS=1" , "AUDITA_TEST_PROVIDER_KEY=" + f . provider . secret )
2026-09-24 02:40:48 +00:00
var stdout , stderr bytes . Buffer
2026-09-24 17:48:52 +00:00
command . Stdout , command . Stderr = & stdout , & stderr
err := command . Run ( )
2026-09-24 02:40:48 +00:00
code := 0
if err != nil {
if exit , ok := err . ( * exec . ExitError ) ; ok {
code = exit . ExitCode ( )
} else {
t . Fatal ( err )
}
}
2026-09-24 17:48:52 +00:00
return code , stdout . String ( ) , stderr . String ( )
}
2026-10-03 17:23:28 +00:00
func ( f processFixture ) runConfigView ( t * testing . T , view string ) ( int , string , string ) {
t . Helper ( )
command := exec . Command ( os . Args [ 0 ] , "-test.run=^TestProcessSubprocessHelper$" , "--" , "config" , "print-effective" , "--config" , f . config , "--view" , view )
command . Dir = f . dir
command . Env = append ( os . Environ ( ) , "AUDITA_HELPER_PROCESS=1" , "AUDITA_TEST_PROVIDER_KEY=" + f . provider . secret )
var stdout , stderr bytes . Buffer
command . Stdout , command . Stderr = & stdout , & stderr
err := command . Run ( )
code := 0
if err != nil {
if exit , ok := err . ( * exec . ExitError ) ; ok {
code = exit . ExitCode ( )
} else {
t . Fatal ( err )
}
}
return code , stdout . String ( ) , stderr . String ( )
}
2026-09-24 17:48:52 +00:00
func ( f processFixture ) errors ( t * testing . T ) reporting . ErrorsDocument {
t . Helper ( )
raw , err := os . ReadFile ( filepath . Join ( f . dir , "errors.yml" ) )
2026-09-24 02:40:48 +00:00
if err != nil {
2026-09-24 17:48:52 +00:00
t . Fatal ( err )
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
var document reporting . ErrorsDocument
if err := yaml . Unmarshal ( raw , & document ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
return document
2026-09-24 02:40:48 +00:00
}
2026-09-25 02:56:44 +00:00
func ( f processFixture ) recoveries ( t * testing . T ) reporting . RecoveriesDocument {
t . Helper ( )
raw , err := os . ReadFile ( filepath . Join ( f . dir , "recoveries.yml" ) )
if err != nil {
t . Fatal ( err )
}
var document reporting . RecoveriesDocument
if err := yaml . Unmarshal ( raw , & document ) ; err != nil {
t . Fatal ( err )
}
return document
}
func ( f processFixture ) setSegments ( t * testing . T , texts ... string ) {
t . Helper ( )
segments := make ( [ ] schema . Segment , len ( texts ) )
for i , value := range texts {
segments [ i ] = schema . Segment { ID : i + 1 , Speaker : "A" , Start : float64 ( i * 10 ) , End : float64 ( i * 10 + 1 ) , Text : value , Categories : [ ] string { "dialogue" } }
}
raw , err := json . Marshal ( segments )
if err != nil {
t . Fatal ( err )
}
if err := os . WriteFile ( f . transcript , raw , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
func ( f processFixture ) readTranscript ( t * testing . T , path string ) * schema . Transcript {
t . Helper ( )
raw , err := os . ReadFile ( path )
if err != nil {
t . Fatal ( err )
}
transcript , err := schema . ParseTranscriptJSON ( raw )
if err != nil {
t . Fatal ( err )
}
return transcript
}
func ( f processFixture ) debugArtifacts ( t * testing . T ) ( * schema . Transcript , [ ] map [ string ] json . RawMessage ) {
t . Helper ( )
dirs , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" ) )
if err != nil || len ( dirs ) != 1 {
t . Fatalf ( "expected one debug run: %v %v" , dirs , err )
}
normalized := f . readTranscript ( t , filepath . Join ( dirs [ 0 ] , "normalized-transcript.json" ) )
raw , err := os . ReadFile ( filepath . Join ( dirs [ 0 ] , "correction-ledger.json" ) )
if err != nil {
t . Fatal ( err )
}
var ledger [ ] map [ string ] json . RawMessage
if err := json . Unmarshal ( raw , & ledger ) ; err != nil {
t . Fatal ( err )
}
return normalized , ledger
}
func assertSpanMetadata ( t * testing . T , normalized , output * schema . Transcript ) {
t . Helper ( )
if len ( normalized . Segments ) != len ( output . Segments ) {
t . Fatalf ( "segment membership changed: normalized=%+v output=%+v" , normalized , output )
}
for i := range normalized . Segments {
normalized . Segments [ i ] . Text = output . Segments [ i ] . Text
}
if ! reflect . DeepEqual ( normalized , output ) {
t . Fatalf ( "segment metadata/order changed: normalized=%+v output=%+v" , normalized , output )
}
}
func ledgerDisposition ( entry map [ string ] json . RawMessage ) string {
var value string
_ = json . Unmarshal ( entry [ "disposition" ] , & value )
return value
}
func ledgerHasDecision ( t * testing . T , entry map [ string ] json . RawMessage , field , key string , approved bool ) bool {
t . Helper ( )
var decisions [ ] struct {
ValidatorKey string ` json:"validator_key" `
Approved bool ` json:"approved" `
}
if err := json . Unmarshal ( entry [ field ] , & decisions ) ; err != nil {
t . Fatal ( err )
}
for _ , decision := range decisions {
if decision . ValidatorKey == key && decision . Approved == approved {
return true
}
}
return false
}
func ( f processFixture ) assertNoStructuralRepairs ( t * testing . T , wantCalls int ) {
t . Helper ( )
dirs , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" ) )
if err != nil || len ( dirs ) != 1 {
t . Fatalf ( "debug run: %v %v" , dirs , err )
}
raw , err := os . ReadFile ( filepath . Join ( dirs [ 0 ] , "llm-execution.json" ) )
if err != nil {
t . Fatal ( err )
}
var execution struct {
Calls [ ] struct {
Summary struct {
ProviderAttempts int ` json:"provider_attempts" `
StructuralRepairs int ` json:"structural_repairs" `
} ` json:"summary" `
} ` json:"calls" `
}
if err := json . Unmarshal ( raw , & execution ) ; err != nil {
t . Fatal ( err )
}
if len ( execution . Calls ) != wantCalls || f . provider . callCount ( ) != wantCalls {
t . Fatalf ( "unexpected generation counts: calls=%d provider=%d" , len ( execution . Calls ) , f . provider . callCount ( ) )
}
for _ , call := range execution . Calls {
if call . Summary . ProviderAttempts != 1 || call . Summary . StructuralRepairs != 0 {
t . Fatalf ( "unexpected structural repair: %+v" , call )
}
}
}
2026-09-25 11:34:26 +00:00
func ( f processFixture ) validationDiagnostics ( t * testing . T ) [ ] map [ string ] json . RawMessage {
t . Helper ( )
paths , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" , "*" , "*-response-payload.json" ) )
if err != nil {
t . Fatal ( err )
}
var diagnostics [ ] map [ string ] json . RawMessage
for _ , path := range paths {
raw , err := os . ReadFile ( path )
if err != nil {
t . Fatal ( err )
}
var payload map [ string ] json . RawMessage
if err := json . Unmarshal ( raw , & payload ) ; err != nil {
t . Fatal ( err )
}
if _ , validation := payload [ "reconciliation" ] ; validation {
diagnostics = append ( diagnostics , payload )
}
}
return diagnostics
}
func ( f processFixture ) assertDecisionRequestsUseOneAttempt ( t * testing . T ) [ ] [ ] int {
t . Helper ( )
f . provider . mu . Lock ( )
requests := append ( [ ] [ ] int ( nil ) , f . provider . validationRequests ... )
providerCalls , proposals , validations := f . provider . calls , f . provider . proposalCalls , f . provider . validationCalls
f . provider . mu . Unlock ( )
if proposals != 1 || validations != len ( requests ) || validations == 0 || providerCalls != proposals + validations {
t . Fatalf ( "provider requests: total=%d proposals=%d validations=%d identities=%v" , providerCalls , proposals , validations , requests )
}
paths , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" , "llm-execution.json" ) )
if err != nil || len ( paths ) != 1 {
t . Fatalf ( "execution path: %v %v" , paths , err )
}
raw , err := os . ReadFile ( paths [ 0 ] )
if err != nil {
t . Fatal ( err )
}
var execution struct {
Calls [ ] struct {
Summary struct {
ProviderAttempts int ` json:"provider_attempts" `
StructuralRepairs int ` json:"structural_repairs" `
} ` json:"summary" `
} ` json:"calls" `
}
if err := json . Unmarshal ( raw , & execution ) ; err != nil {
t . Fatal ( err )
}
if len ( execution . Calls ) != providerCalls {
t . Fatalf ( "logical/provider calls: %d/%d" , len ( execution . Calls ) , providerCalls )
}
for _ , call := range execution . Calls {
if call . Summary . ProviderAttempts != 1 || call . Summary . StructuralRepairs != 0 {
t . Fatalf ( "extra generation or structural repair: %+v" , call )
}
}
metadataPaths , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" , "*" , "*-request-metadata.json" ) )
if err != nil {
t . Fatal ( err )
}
validatorsSeen := make ( map [ string ] bool )
for _ , path := range metadataPaths {
raw , err := os . ReadFile ( path )
if err != nil {
t . Fatal ( err )
}
var metadata struct {
ValidatorName string ` json:"validator_name" `
BatchIndex int ` json:"batch_index" `
}
if err := json . Unmarshal ( raw , & metadata ) ; err != nil {
t . Fatal ( err )
}
if metadata . ValidatorName == "" {
continue
}
if validatorsSeen [ metadata . ValidatorName ] || metadata . BatchIndex != 0 {
t . Fatalf ( "duplicate or split validation call: %+v" , metadata )
}
validatorsSeen [ metadata . ValidatorName ] = true
}
if len ( validatorsSeen ) != validations {
t . Fatalf ( "validator operations=%d requests=%d" , len ( validatorsSeen ) , validations )
}
return requests
}
func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse ( t * testing . T ) {
f := newProcessFixture ( t , "decision-mixed" , "[grammar]" )
f . setSegments ( t , "teh" , "cat" , "bad" , "slow" )
code , stdout , stderr := f . run ( t , true , "--debug" )
if code != 0 || stdout != "" || ! strings . Contains ( stderr , "1 corrections applied; 1 work units with errors; 0 recoveries" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output := f . readTranscript ( t , f . output )
normalized , ledger := f . debugArtifacts ( t )
assertSpanMetadata ( t , normalized , output )
if len ( output . Segments ) != 4 || output . Segments [ 0 ] . Text != "the" || output . Segments [ 1 ] . Text != "cat" || output . Segments [ 2 ] . Text != "bad" || output . Segments [ 3 ] . Text != "slow" {
t . Fatalf ( "partial validation transcript: %+v" , output )
}
errors , recoveries := f . errors ( t ) , f . recoveries ( t )
if errors . Status != reporting . StatusCompleted || len ( errors . Errors ) != 1 || len ( errors . Errors [ 0 ] . Failures ) != 1 || errors . Errors [ 0 ] . Failures [ 0 ] . Reason != "validation_decisions_invalid" || ! reflect . DeepEqual ( errors . Errors [ 0 ] . Failures [ 0 ] . ProposalIndices , [ ] int { 2 , 3 } ) || * errors . Summary . CorrectionsApplied != 1 || * errors . Summary . WorkUnitsWithErrors != 1 || * errors . Summary . Recoveries != 0 || len ( recoveries . Recoveries ) != 0 || * recoveries . Summary . ProposalResponsesWithFiltering != 0 {
t . Fatalf ( "partial validation reports: errors=%+v recoveries=%+v" , errors , recoveries )
}
if len ( ledger ) != 4 {
t . Fatalf ( "unexpected candidate ledger entries: %+v" , ledger )
}
for index , want := range [ ] struct { disposition , reason string } { { "applied" , "" } , { "rejected" , "llm_rejected" } , { "unavailable" , "missing_validation_decision" } , { "unavailable" , "duplicate_validation_decision" } } {
entry := ledger [ index ]
var candidate int
var reason string
if err := json . Unmarshal ( entry [ "proposal_index" ] , & candidate ) ; err != nil {
t . Fatal ( err )
}
if err := json . Unmarshal ( entry [ "disposition_reason_code" ] , & reason ) ; err != nil && len ( entry [ "disposition_reason_code" ] ) > 0 {
t . Fatal ( err )
}
if candidate != index || ledgerDisposition ( entry ) != want . disposition || reason != want . reason || ( index != 0 && entry [ "applied_corrected_text" ] != nil ) {
t . Fatalf ( "candidate %d evidence: %+v" , index , entry )
}
if index > 0 {
var decisions [ ] struct {
Disposition string ` json:"disposition" `
}
if err := json . Unmarshal ( entry [ "llm_validator_decisions" ] , & decisions ) ; err != nil || len ( decisions ) != 1 || decisions [ 0 ] . Disposition != want . disposition {
t . Fatalf ( "candidate %d decision: %+v %v" , index , decisions , err )
}
}
}
var mixedSeen bool
for _ , diagnostic := range f . validationDiagnostics ( t ) {
var reconciliation struct {
UnavailableIndices [ ] int ` json:"unavailable_indices" `
IgnoredIndices [ ] int ` json:"ignored_indices" `
Decisions [ ] struct {
ProposalIndex int ` json:"proposal_index" `
ReasonCode string ` json:"reason_code" `
} ` json:"decisions" `
}
if err := json . Unmarshal ( diagnostic [ "reconciliation" ] , & reconciliation ) ; err != nil {
t . Fatal ( err )
}
if ! reflect . DeepEqual ( reconciliation . UnavailableIndices , [ ] int { 2 , 3 } ) {
continue
}
var decoded struct {
Validations [ ] struct {
CorrectionIndex int ` json:"correction_index" `
} ` json:"validations" `
}
if err := json . Unmarshal ( diagnostic [ "decoded" ] , & decoded ) ; err != nil {
t . Fatal ( err )
}
var finalContent string
if err := json . Unmarshal ( diagnostic [ "final_content" ] , & finalContent ) ; err != nil {
t . Fatal ( err )
}
if ! reflect . DeepEqual ( reconciliation . IgnoredIndices , [ ] int { 99 } ) || len ( decoded . Validations ) != 5 || len ( reconciliation . Decisions ) != 4 || reconciliation . Decisions [ 2 ] . ProposalIndex != 2 || reconciliation . Decisions [ 2 ] . ReasonCode != "missing_validation_decision" || reconciliation . Decisions [ 3 ] . ProposalIndex != 3 || reconciliation . Decisions [ 3 ] . ReasonCode != "duplicate_validation_decision" || finalContent == "" {
t . Fatalf ( "mixed response evidence: %+v %+v" , reconciliation , decoded )
}
mixedSeen = true
}
if ! mixedSeen {
t . Fatal ( "missing mixed decision interaction evidence" )
}
requests := f . assertDecisionRequestsUseOneAttempt ( t )
full , survivor := 0 , 0
for _ , indices := range requests {
switch {
2026-09-25 12:38:23 +00:00
case reflect . DeepEqual ( indices , [ ] int { 1 , 2 , 3 , 4 } ) :
2026-09-25 11:34:26 +00:00
full ++
2026-09-25 12:38:23 +00:00
case reflect . DeepEqual ( indices , [ ] int { 1 } ) :
2026-09-25 11:34:26 +00:00
survivor ++
default :
t . Fatalf ( "unexpected validation identities: %v" , requests )
}
}
if full != 1 || survivor == 0 {
t . Fatalf ( "wrong validation progression: %v" , requests )
}
}
func TestSubprocessReportsFilteredExtraValidatorDecision ( t * testing . T ) {
f := newProcessFixture ( t , "decision-extra-only" , "[grammar]" )
f . setSegments ( t , "teh" )
code , stdout , stderr := f . run ( t , true , "--debug" )
if code != 0 || stdout != "" || ! strings . Contains ( stderr , "1 corrections applied; 0 work units with errors; 1 recoveries" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output := f . readTranscript ( t , f . output )
if len ( output . Segments ) != 1 || output . Segments [ 0 ] . Text != "the" {
t . Fatalf ( "lost usable correction: %+v" , output )
}
errors , recoveries := f . errors ( t ) , f . recoveries ( t )
if len ( errors . Errors ) != 0 || len ( recoveries . Recoveries ) != 1 || ! reflect . DeepEqual ( recoveries . Recoveries [ 0 ] . Reasons , [ ] string { "validation_decisions_filtered" } ) || recoveries . Recoveries [ 0 ] . Operation != "validation" || len ( recoveries . Recoveries [ 0 ] . FilteredProposals ) != 0 || * recoveries . Summary . CorrectionsApplied != 1 || * recoveries . Summary . Recoveries != 1 || * recoveries . Summary . ProposalResponsesWithFiltering != 0 {
t . Fatalf ( "extra-only reports: errors=%+v recoveries=%+v" , errors , recoveries )
}
markdown , err := os . ReadFile ( filepath . Join ( f . dir , "report.md" ) )
if err != nil {
t . Fatal ( err )
}
if ! bytes . Contains ( markdown , [ ] byte ( "Corrections applied: 1" ) ) || ! bytes . Contains ( markdown , [ ] byte ( "Errors: 0 work units" ) ) || ! bytes . Contains ( markdown , [ ] byte ( "Recoveries: 1 of " ) ) {
t . Fatalf ( "Markdown totals disagree: %s" , markdown )
}
_ , ledger := f . debugArtifacts ( t )
if len ( ledger ) != 1 || ledgerDisposition ( ledger [ 0 ] ) != "applied" {
t . Fatalf ( "unknown candidate in ledger: %+v" , ledger )
}
ignored := 0
for _ , diagnostic := range f . validationDiagnostics ( t ) {
var reconciliation struct {
IgnoredIndices [ ] int ` json:"ignored_indices" `
}
if err := json . Unmarshal ( diagnostic [ "reconciliation" ] , & reconciliation ) ; err != nil {
t . Fatal ( err )
}
if reflect . DeepEqual ( reconciliation . IgnoredIndices , [ ] int { 99 } ) {
ignored ++
}
}
if ignored != 1 {
t . Fatalf ( "expected one handled extra decision, got %d" , ignored )
}
requests := f . assertDecisionRequestsUseOneAttempt ( t )
for _ , indices := range requests {
2026-09-25 12:38:23 +00:00
if ! reflect . DeepEqual ( indices , [ ] int { 1 } ) {
2026-09-25 11:34:26 +00:00
t . Fatalf ( "unexpected identities: %v" , requests )
}
}
}
2026-09-25 02:56:44 +00:00
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup ( t * testing . T ) {
f := newProcessFixture ( t , "span-clean" , "[spoken_word]" )
f . setSegments ( t , "uh hello" , "I I think" )
code , stdout , stderr := f . run ( t , true , "--debug" )
if code != 0 || stdout != "" || ! strings . Contains ( stderr , "2 corrections applied; 0 work units with errors; 0 recoveries" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output := f . readTranscript ( t , f . output )
normalized , ledger := f . debugArtifacts ( t )
assertSpanMetadata ( t , normalized , output )
if len ( output . Segments ) != 2 || output . Segments [ 0 ] . Text != "hello" || output . Segments [ 1 ] . Text != "I think" {
t . Fatalf ( "unexpected corrected transcript: %+v" , output )
}
if len ( ledger ) != 2 || ledgerDisposition ( ledger [ 0 ] ) != "applied" || ledgerDisposition ( ledger [ 1 ] ) != "applied" || string ( ledger [ 0 ] [ "proposed_corrected_text" ] ) != ` "" ` || string ( ledger [ 0 ] [ "applied_corrected_text" ] ) != ` "" ` {
t . Fatalf ( "deletion missing from ledger: %+v" , ledger )
}
errors , recoveries := f . errors ( t ) , f . recoveries ( t )
if len ( errors . Errors ) != 0 || len ( recoveries . Recoveries ) != 0 || * errors . Summary . CorrectionsApplied != 2 || * errors . Summary . WorkUnitsCompleted != 1 || * recoveries . Summary . CorrectionsApplied != 2 {
t . Fatalf ( "report totals disagree: errors=%+v recoveries=%+v" , errors , recoveries )
}
f . assertNoStructuralRepairs ( t , 3 )
}
func TestSubprocessAppliesWhitespaceOnlySpan ( t * testing . T ) {
f := newProcessFixture ( t , "span-spacing" , "[grammar]" )
f . setSegments ( t , "hello world" )
code , _ , stderr := f . run ( t , true , "--debug" )
if code != 0 || ! strings . Contains ( stderr , "1 corrections applied; 0 work units with errors; 0 recoveries" ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
}
output := f . readTranscript ( t , f . output )
normalized , _ := f . debugArtifacts ( t )
assertSpanMetadata ( t , normalized , output )
if len ( output . Segments ) != 1 || output . Segments [ 0 ] . Text != "hello world" || len ( f . errors ( t ) . Errors ) != 0 || len ( f . recoveries ( t ) . Recoveries ) != 0 {
t . Fatalf ( "spacing correction failed: %+v" , output )
}
f . assertNoStructuralRepairs ( t , 3 )
}
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections ( t * testing . T ) {
f := newProcessFixture ( t , "span-filter" , "[spoken_word]" )
f . setSegments ( t , "uh hello" , "I I think" , "uh" )
code , _ , stderr := f . run ( t , true , "--debug" )
if code != 0 || ! strings . Contains ( stderr , "2 corrections applied; 0 work units with errors; 1 recoveries" ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
}
output := f . readTranscript ( t , f . output )
normalized , ledger := f . debugArtifacts ( t )
assertSpanMetadata ( t , normalized , output )
if len ( output . Segments ) != 3 || output . Segments [ 0 ] . Text != "hello" || output . Segments [ 1 ] . Text != "I think" || output . Segments [ 2 ] . Text != "uh" {
t . Fatalf ( "filtered segment or survivors wrong: %+v" , output )
}
recoveries := f . recoveries ( t )
if len ( f . errors ( t ) . Errors ) != 0 || len ( recoveries . Recoveries ) != 1 || recoveries . Recoveries [ 0 ] . Operation != "proposal" || len ( recoveries . Recoveries [ 0 ] . Reasons ) != 1 || recoveries . Recoveries [ 0 ] . Reasons [ 0 ] != "proposals_filtered" || len ( recoveries . Recoveries [ 0 ] . FilteredProposals ) != 1 || recoveries . Recoveries [ 0 ] . FilteredProposals [ 0 ] . Reason != "empty_resulting_segment" || * recoveries . Summary . ProposalResponsesWithFiltering != 1 {
t . Fatalf ( "filtering recovery wrong: %+v" , recoveries )
}
if len ( ledger ) != 3 || ledgerDisposition ( ledger [ 2 ] ) != "rejected" || string ( ledger [ 2 ] [ "proposed_corrected_text" ] ) != ` "" ` {
t . Fatalf ( "filtered deletion evidence missing: %+v" , ledger )
}
if ! ledgerHasDecision ( t , ledger [ 2 ] , "deterministic_validator_decisions" , "non_empty_resulting_segment" , false ) {
t . Fatalf ( "renamed validator missing from evidence: %+v" , ledger [ 2 ] )
}
f . assertNoStructuralRepairs ( t , 3 )
}
func TestSubprocessRespectsNegativeMeaningReviewOnDeletion ( t * testing . T ) {
f := newProcessFixture ( t , "span-negative" , "[grammar]" )
f . setSegments ( t , "not good" )
code , _ , stderr := f . run ( t , true , "--debug" )
if code != 0 || ! strings . Contains ( stderr , "0 corrections applied; 0 work units with errors; 0 recoveries" ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
}
output := f . readTranscript ( t , f . output )
normalized , ledger := f . debugArtifacts ( t )
assertSpanMetadata ( t , normalized , output )
if len ( output . Segments ) != 1 || output . Segments [ 0 ] . Text != "not good" || len ( f . errors ( t ) . Errors ) != 0 || len ( f . recoveries ( t ) . Recoveries ) != 0 {
t . Fatalf ( "negative review changed outcome: %+v" , output )
}
if len ( ledger ) != 1 || ledgerDisposition ( ledger [ 0 ] ) != "rejected" || string ( ledger [ 0 ] [ "proposed_corrected_text" ] ) != ` "" ` {
t . Fatalf ( "rejection evidence missing: %+v" , ledger )
}
if _ , present := ledger [ 0 ] [ "applied_corrected_text" ] ; present || ! ledgerHasDecision ( t , ledger [ 0 ] , "llm_validator_decisions" , "editorial_review" , true ) || ! ledgerHasDecision ( t , ledger [ 0 ] , "llm_validator_decisions" , "meaning_reversal_review" , false ) {
t . Fatalf ( "negative decision not recorded: %+v" , ledger [ 0 ] )
}
f . assertNoStructuralRepairs ( t , 3 )
}
2026-09-24 17:48:52 +00:00
func TestSubprocessDefaultFileOutputAndReports ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
code , stdout , stderr := f . run ( t , true )
if code != 0 || stdout != "" || ! strings . HasPrefix ( stderr , "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries." ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "output: %s %v" , output , err )
}
document := f . errors ( t )
if document . Status != reporting . StatusCompleted || document . Summary . WorkUnitsCompleted == nil || * document . Summary . WorkUnitsCompleted != 1 || len ( document . Errors ) != 0 {
t . Fatalf ( "errors document: %+v" , document )
}
for _ , name := range [ ] string { "errors.yml" , "recoveries.yml" , "report.md" } {
2026-09-24 18:03:25 +00:00
report , err := os . ReadFile ( filepath . Join ( f . dir , name ) )
if err != nil {
2026-09-24 17:48:52 +00:00
t . Fatal ( err )
}
2026-09-24 18:03:25 +00:00
if bytes . Contains ( report , [ ] byte ( "hello world" ) ) || bytes . Contains ( report , [ ] byte ( "Hello world" ) ) || bytes . Contains ( report , [ ] byte ( f . provider . secret ) ) {
t . Fatalf ( "ordinary report contains payload or secret: %s" , name )
}
2026-09-24 17:48:52 +00:00
}
if _ , err := os . Stat ( filepath . Join ( f . dir , "runs" ) ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "debug created by default: %v" , err )
}
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
func TestSubprocessStdoutAndSelectedReportDirectory ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
reports := filepath . Join ( f . dir , "reports" )
if err := os . Mkdir ( reports , 0700 ) ; err != nil {
t . Fatal ( err )
}
code , stdout , stderr := f . run ( t , false , "--report-dir" , reports )
if code != 0 || ! strings . Contains ( stdout , "Hello world" ) || ! strings . HasPrefix ( stderr , "Completed:" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
if _ , err := os . Stat ( f . output ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "stdout run wrote transcript file: %v" , err )
}
if _ , err := os . Stat ( filepath . Join ( reports , "errors.yml" ) ) ; err != nil {
t . Fatal ( err )
2026-09-24 02:40:48 +00:00
}
}
2026-10-03 13:53:05 +00:00
func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
transcriptDir := filepath . Join ( f . dir , "transcripts" )
reportDir := filepath . Join ( f . dir , "reports" )
debugDir := filepath . Join ( f . dir , "debug" )
for _ , dir := range [ ] string { transcriptDir , reportDir } {
if err := os . Mkdir ( dir , 0700 ) ; err != nil {
t . Fatal ( err )
}
}
f . output = filepath . Join ( transcriptDir , "corrected.json" )
code , stdout , stderr := f . run ( t , true , "--report-dir" , reportDir , "--debug" , "--work-dir" , debugDir )
if code != 0 || stdout != "" || ! strings . HasPrefix ( stderr , "Completed:" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
if output , err := os . ReadFile ( f . output ) ; err != nil || ! bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "transcript: %s %v" , output , err )
}
for _ , name := range [ ] string { "errors.yml" , "recoveries.yml" , "report.md" } {
if _ , err := os . Stat ( filepath . Join ( reportDir , name ) ) ; err != nil {
t . Fatalf ( "report %s: %v" , name , err )
}
if _ , err := os . Stat ( filepath . Join ( transcriptDir , name ) ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "report %s appeared beside transcript: %v" , name , err )
}
}
dirs , err := filepath . Glob ( filepath . Join ( debugDir , "run-*" ) )
if err != nil || len ( dirs ) != 1 {
t . Fatalf ( "debug runs: %v %v" , dirs , err )
}
if _ , err := os . Stat ( filepath . Join ( dirs [ 0 ] , "correction-ledger.json" ) ) ; err != nil {
t . Fatalf ( "debug capture: %v" , err )
}
}
2026-09-24 17:48:52 +00:00
func TestSubprocessReportingAndDebugSelections ( t * testing . T ) {
2026-09-24 18:03:25 +00:00
var expectedOutput , expectedSummary string
for _ , selection := range [ ] struct { reporting , debug bool } {
{ false , false } , { false , true } , { true , false } , { true , true } ,
} {
2026-09-24 17:48:52 +00:00
f := newProcessFixture ( t , "normal" , "[grammar]" )
2026-10-03 17:23:28 +00:00
if selection . reporting && selection . debug {
original , err := os . ReadFile ( f . config )
if err != nil {
t . Fatal ( err )
}
old := fmt . Sprintf ( "backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY" , f . server . URL )
inherited := fmt . Sprintf ( "backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1" , f . server . URL )
updated := strings . Replace ( string ( original ) , old , inherited , 1 )
if updated == string ( original ) {
t . Fatal ( "process fixture backend configuration was not found" )
}
if err := os . WriteFile ( f . config , [ ] byte ( updated ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
2026-10-03 13:46:50 +00:00
reportDir := filepath . Join ( f . dir , "missing" )
extra := [ ] string { "--reporting=false" , "--report-dir" , reportDir }
2026-09-24 18:03:25 +00:00
if selection . reporting {
2026-10-03 13:46:50 +00:00
reportDir = filepath . Join ( f . dir , "selected-reports" )
if err := os . Mkdir ( reportDir , 0700 ) ; err != nil {
t . Fatal ( err )
}
extra = [ ] string { "--reporting=true" , "--report-dir" , reportDir }
2026-09-24 18:03:25 +00:00
}
if selection . debug {
2026-09-24 17:48:52 +00:00
extra = append ( extra , "--debug" )
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
code , stdout , stderr := f . run ( t , true , extra ... )
if code != 0 || stdout != "" || ! strings . HasPrefix ( stderr , "Completed:" ) {
2026-09-24 18:03:25 +00:00
t . Fatalf ( "selection=%+v code=%d stdout=%q stderr=%q" , selection , code , stdout , stderr )
}
output , err := os . ReadFile ( f . output )
if err != nil {
t . Fatal ( err )
}
summary := strings . SplitN ( stderr , "\n" , 2 ) [ 0 ]
if expectedOutput == "" {
expectedOutput , expectedSummary = string ( output ) , summary
} else if string ( output ) != expectedOutput || summary != expectedSummary {
t . Fatalf ( "selection changed processing: %+v summary=%q output=%s" , selection , summary , output )
2026-09-24 02:40:48 +00:00
}
2026-09-24 18:03:25 +00:00
for _ , name := range [ ] string { "errors.yml" , "recoveries.yml" , "report.md" } {
2026-10-03 13:46:50 +00:00
_ , err := os . Stat ( filepath . Join ( reportDir , name ) )
2026-09-24 18:03:25 +00:00
if selection . reporting && err != nil || ! selection . reporting && ! os . IsNotExist ( err ) {
t . Fatalf ( "selection=%+v report %s: %v" , selection , name , err )
}
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
dirs , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" ) )
2026-09-24 18:03:25 +00:00
if err != nil || len ( dirs ) != boolInt ( selection . debug ) {
t . Fatalf ( "selection=%+v dirs=%v %v" , selection , dirs , err )
2026-09-24 02:40:48 +00:00
}
2026-09-24 18:03:25 +00:00
if selection . debug {
2026-09-24 17:48:52 +00:00
for _ , name := range [ ] string { "source-transcript.json" , "normalized-transcript.json" , "correction-ledger.json" , "llm-execution.json" } {
if _ , err := os . Stat ( filepath . Join ( dirs [ 0 ] , name ) ) ; err != nil {
2026-09-24 02:40:48 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
}
2026-10-03 17:23:28 +00:00
if selection . reporting {
effective , err := os . ReadFile ( filepath . Join ( dirs [ 0 ] , "effective-config.redacted.yaml" ) )
if err != nil {
t . Fatal ( err )
}
if ! strings . Contains ( string ( effective ) , f . server . URL + "/v1" ) || ! strings . Contains ( string ( effective ) , "concurrency: 1" ) || ! strings . Contains ( string ( effective ) , "queue_capacity: 0" ) || strings . Contains ( string ( effective ) , f . provider . secret ) {
t . Fatalf ( "effective inherited settings: %s" , effective )
}
origins , err := os . ReadFile ( filepath . Join ( dirs [ 0 ] , "config-origins.yaml" ) )
if err != nil {
t . Fatal ( err )
}
for _ , want := range [ ] string { "from: /backends/local-base/endpoint" , "to: /backends/local/endpoint" , "reference: /backends/local/extends" } {
if ! strings . Contains ( string ( origins ) , want ) {
t . Fatalf ( "configuration origins missing %q: %s" , want , origins )
}
}
if strings . Contains ( string ( origins ) , f . provider . secret ) {
t . Fatal ( "configuration origins exposed the resolved inherited credential" )
}
code , explain , stderr := f . runConfigView ( t , "explain" )
if code != 0 || stderr != "" || ! strings . Contains ( explain , "from: /backends/local-base/endpoint" ) || strings . Contains ( explain , f . provider . secret ) {
t . Fatalf ( "inheritance explanation: code=%d stdout=%s stderr=%s" , code , explain , stderr )
}
}
2026-09-24 02:40:48 +00:00
}
2026-09-24 17:48:52 +00:00
}
2026-09-24 02:40:48 +00:00
}
2026-09-24 15:32:13 +00:00
2026-09-24 17:48:52 +00:00
func boolInt ( value bool ) int {
if value {
return 1
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
return 0
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
func TestSubprocessAllModelFailureStillCompletes ( t * testing . T ) {
2026-09-24 18:03:25 +00:00
f := newProcessFixture ( t , "all-model-failure" , "[grammar, grammar]" )
2026-09-24 17:48:52 +00:00
code , stdout , stderr := f . run ( t , true )
2026-09-24 18:03:25 +00:00
if code != 0 || stdout != "" || ! strings . Contains ( stderr , "2 work units with errors" ) {
2026-09-24 17:48:52 +00:00
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "hello world" ) ) {
t . Fatalf ( "normalized output: %s %v" , output , err )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
document := f . errors ( t )
2026-09-24 18:03:25 +00:00
if document . Status != reporting . StatusCompleted || len ( document . Errors ) != 2 || document . Errors [ 0 ] . Scope != "work_unit" || * document . Summary . WorkUnitsWithErrors != 2 || document . Errors [ 0 ] . ModuleInstance == document . Errors [ 1 ] . ModuleInstance || f . provider . callCount ( ) != 2 {
2026-09-24 17:48:52 +00:00
t . Fatalf ( "model failure report: %+v" , document )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
if strings . Contains ( stderr , f . provider . secret ) {
t . Fatal ( "secret leaked in stderr" )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
}
2026-09-24 18:03:25 +00:00
func TestSubprocessAllValidationFailureStillCompletes ( t * testing . T ) {
f := newProcessFixture ( t , "all-validation-failure" , "[grammar, grammar]" )
code , stdout , stderr := f . run ( t , true )
if code != 0 || stdout != "" || ! strings . Contains ( stderr , "2 work units with errors" ) {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "hello world" ) ) || bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "unchecked output: %s %v" , output , err )
}
document := f . errors ( t )
if document . Status != reporting . StatusCompleted || len ( document . Errors ) != 2 || * document . Summary . WorkUnitsWithErrors != 2 || document . Errors [ 0 ] . ModuleInstance == document . Errors [ 1 ] . ModuleInstance || f . provider . callCount ( ) != 4 {
t . Fatalf ( "validation failure report: %+v; calls=%d" , document , f . provider . callCount ( ) )
}
}
func TestSubprocessRecoveredFilteringSurvivesLaterFailure ( t * testing . T ) {
f := newProcessFixture ( t , "repair-filter-later-failure" , "[grammar, grammar]" )
code , _ , stderr := f . run ( t , true )
if code != 0 || ! strings . Contains ( stderr , "1 corrections applied; 1 work units with errors; 1 recoveries" ) {
t . Fatalf ( "code=%d stderr=%q errors=%+v" , code , stderr , f . errors ( t ) )
}
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "output: %s %v" , output , err )
}
errors := f . errors ( t )
if len ( errors . Errors ) != 1 || errors . Errors [ 0 ] . ModuleInstance != "grammar_2" {
t . Fatalf ( "later failure: %+v" , errors )
}
raw , err := os . ReadFile ( filepath . Join ( f . dir , "recoveries.yml" ) )
if err != nil {
t . Fatal ( err )
}
var recoveries reporting . RecoveriesDocument
if err := yaml . Unmarshal ( raw , & recoveries ) ; err != nil {
t . Fatal ( err )
}
if len ( recoveries . Recoveries ) != 1 || recoveries . Recoveries [ 0 ] . ModuleInstance != "grammar_1" || recoveries . Recoveries [ 0 ] . Operation != "proposal" || len ( recoveries . Recoveries [ 0 ] . FilteredProposals ) != 1 || * recoveries . Summary . Recoveries != 1 {
t . Fatalf ( "combined recovery: %+v" , recoveries )
}
if ! strings . Contains ( strings . Join ( recoveries . Recoveries [ 0 ] . Reasons , "," ) , "output_repaired" ) || ! strings . Contains ( strings . Join ( recoveries . Recoveries [ 0 ] . Reasons , "," ) , "proposals_filtered" ) {
t . Fatalf ( "recovery reasons: %+v" , recoveries . Recoveries [ 0 ] )
}
}
func TestSubprocessRepairedInvalidDecisionsRemainAnError ( t * testing . T ) {
f := newProcessFixture ( t , "invalid-decision-after-repair" , "[grammar]" )
code , _ , stderr := f . run ( t , true )
if code != 0 || ! strings . Contains ( stderr , "0 corrections applied; 1 work units with errors; 0 recoveries" ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
}
document := f . errors ( t )
if document . Status != reporting . StatusCompleted || len ( document . Errors ) != 1 || len ( document . Errors [ 0 ] . Failures ) != 1 || document . Errors [ 0 ] . Failures [ 0 ] . Reason != "validation_decisions_invalid" {
t . Fatalf ( "invalid decisions report: %+v" , document )
}
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "hello world" ) ) || bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "unchecked output: %s %v" , output , err )
}
}
func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript ( t * testing . T ) {
f := newProcessFixture ( t , "wait-cancel" , "[grammar, grammar]" )
ctx , stop := context . WithTimeout ( context . Background ( ) , 5 * time . Second )
defer stop ( )
args := [ ] string { "-test.run=^TestProcessSubprocessHelper$" , "--" , "process" , f . transcript , "--config" , f . config , "--glossary" , f . glossary , "--output" , f . output }
command := exec . CommandContext ( ctx , os . Args [ 0 ] , args ... )
command . Env = append ( os . Environ ( ) , "AUDITA_HELPER_PROCESS=1" , "AUDITA_TEST_PROVIDER_KEY=" + f . provider . secret )
var stderr bytes . Buffer
command . Stderr = & stderr
if err := command . Start ( ) ; err != nil {
t . Fatal ( err )
}
select {
case <- f . provider . entered :
case <- ctx . Done ( ) :
t . Fatalf ( "provider was not reached: %v" , ctx . Err ( ) )
}
if err := command . Process . Signal ( os . Interrupt ) ; err != nil {
t . Fatal ( err )
}
err := command . Wait ( )
if ctx . Err ( ) != nil || err == nil {
t . Fatalf ( "cancellation did not stop run: %v %v stderr=%q" , err , ctx . Err ( ) , stderr . String ( ) )
}
if document := f . errors ( t ) ; document . Status != reporting . StatusFailed || len ( document . Errors ) == 0 {
t . Fatalf ( "cancellation report: %+v" , document )
}
if _ , err := os . Stat ( f . output ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "canceled run published transcript: %v" , err )
}
}
2026-09-24 17:48:52 +00:00
func TestSubprocessPartialValidationAndLaterStage ( t * testing . T ) {
f := newProcessFixture ( t , "partial-validation" , "[grammar, grammar]" )
code , _ , stderr := f . run ( t , true )
if code != 0 || ! strings . Contains ( stderr , "1 corrections applied; 1 work units with errors" ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
document := f . errors ( t )
if len ( document . Errors ) != 1 || document . Errors [ 0 ] . ModuleInstance != "grammar_1" || * document . Summary . WorkUnitsCompleted != 2 {
t . Fatalf ( "partial outcome: %+v" , document )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "later correction: %s %v" , output , err )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
}
func TestSubprocessInvalidInputLeavesFailedReports ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
if err := os . WriteFile ( f . transcript , [ ] byte ( "not JSON" ) , 0600 ) ; err != nil {
2026-09-24 15:32:13 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
code , _ , _ := f . run ( t , true )
if code != 1 || f . provider . callCount ( ) != 0 {
t . Fatalf ( "invalid input dispatched: code=%d calls=%d" , code , f . provider . callCount ( ) )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
if document := f . errors ( t ) ; document . Status != reporting . StatusFailed || len ( document . Errors ) != 1 || document . Errors [ 0 ] . Scope != "run" {
t . Fatalf ( "failed report: %+v" , document )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
if _ , err := os . Stat ( f . output ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "failed transcript published: %v" , err )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
var recoveries reporting . RecoveriesDocument
if raw , err := os . ReadFile ( filepath . Join ( f . dir , "recoveries.yml" ) ) ; err != nil {
t . Fatal ( err )
} else if err := yaml . Unmarshal ( raw , & recoveries ) ; err != nil {
2026-09-24 15:32:13 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
if recoveries . Status != reporting . StatusFailed {
t . Fatalf ( "recoveries status: %+v" , recoveries )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
if raw , err := os . ReadFile ( filepath . Join ( f . dir , "report.md" ) ) ; err != nil || ! bytes . Contains ( raw , [ ] byte ( "Status: failed" ) ) {
t . Fatalf ( "Markdown status: %s %v" , raw , err )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
}
2026-10-03 17:23:28 +00:00
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
original , err := os . ReadFile ( f . config )
if err != nil {
t . Fatal ( err )
}
updated := strings . Replace ( string ( original ) , " local:\n endpoint:" , " local:\n extends: missing\n endpoint:" , 1 )
if updated == string ( original ) {
t . Fatal ( "process fixture backend configuration was not found" )
}
if err := os . WriteFile ( f . config , [ ] byte ( updated ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
code , stdout , stderr := f . run ( t , true )
if code != 2 || stdout != "" || f . provider . callCount ( ) != 0 || ! strings . Contains ( stderr , "referenced sibling does not exist" ) {
t . Fatalf ( "missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d" , code , stdout , stderr , f . provider . callCount ( ) )
}
}
2026-10-03 13:53:05 +00:00
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
if err := os . WriteFile ( f . transcript , [ ] byte ( "not JSON" ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
reportDir := filepath . Join ( f . dir , "failed-reports" )
if err := os . Mkdir ( reportDir , 0700 ) ; err != nil {
t . Fatal ( err )
}
code , _ , _ := f . run ( t , true , "--report-dir" , reportDir )
if code != 1 || f . provider . callCount ( ) != 0 {
t . Fatalf ( "invalid input dispatched: code=%d calls=%d" , code , f . provider . callCount ( ) )
}
raw , err := os . ReadFile ( filepath . Join ( reportDir , "errors.yml" ) )
if err != nil {
t . Fatal ( err )
}
var document reporting . ErrorsDocument
if err := yaml . Unmarshal ( raw , & document ) ; err != nil {
t . Fatal ( err )
}
if document . Status != reporting . StatusFailed || len ( document . Errors ) != 1 || document . Errors [ 0 ] . Scope != "run" {
t . Fatalf ( "failed report: %+v" , document )
}
if _ , err := os . Stat ( filepath . Join ( reportDir , "recoveries.yml" ) ) ; err != nil {
t . Fatal ( err )
}
if raw , err := os . ReadFile ( filepath . Join ( reportDir , "report.md" ) ) ; err != nil || ! bytes . Contains ( raw , [ ] byte ( "Status: failed" ) ) {
t . Fatalf ( "Markdown status: %s %v" , raw , err )
}
if _ , err := os . Stat ( filepath . Join ( f . dir , "errors.yml" ) ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "failed report appeared in default directory: %v" , err )
}
if _ , err := os . Stat ( f . output ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "failed transcript published: %v" , err )
}
}
2026-09-24 17:48:52 +00:00
func TestSubprocessOutputFailureBeforeModelDispatch ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
f . output = filepath . Join ( f . dir , "missing" , "output.json" )
code , _ , stderr := f . run ( t , true )
if code == 0 || f . provider . callCount ( ) != 0 || ! strings . Contains ( stderr , "report directory" ) {
t . Fatalf ( "preflight: code=%d calls=%d stderr=%q" , code , f . provider . callCount ( ) , stderr )
}
if _ , err := os . Stat ( filepath . Join ( f . dir , "errors.yml" ) ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "report initialized before preflight: %v" , err )
}
}
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
code , _ , stderr := f . run ( t , true , "--debug" , "--work-dir" , f . transcript )
if code != 0 || ! strings . HasPrefix ( stderr , "Completed: 1 corrections applied" ) || ! strings . Contains ( stderr , "debug directory" ) {
t . Fatalf ( "optional debug failure changed processing: code=%d stderr=%q" , code , stderr )
}
if _ , err := os . Stat ( f . output ) ; err != nil {
2026-09-24 15:32:13 +00:00
t . Fatal ( err )
}
2026-09-24 17:48:52 +00:00
if document := f . errors ( t ) ; document . Status != reporting . StatusCompleted {
t . Fatalf ( "report status: %+v" , document )
2026-09-24 15:32:13 +00:00
}
2026-09-24 17:48:52 +00:00
}
func TestSubprocessTranscriptPreflightWithoutReporting ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
f . output = filepath . Join ( f . dir , "missing" , "output.json" )
code , _ , stderr := f . run ( t , true , "--reporting=false" )
if code == 0 || f . provider . callCount ( ) != 0 || ! strings . Contains ( stderr , "transcript output parent" ) {
t . Fatalf ( "transcript preflight: code=%d calls=%d stderr=%q" , code , f . provider . callCount ( ) , stderr )
2026-09-24 15:32:13 +00:00
}
}
2026-09-24 21:48:18 +00:00
func TestSubprocessOutputPermissionsHonorUmask ( t * testing . T ) {
for _ , mask := range [ ] string { "0077" , "0022" } {
t . Run ( mask , func ( t * testing . T ) {
t . Setenv ( "AUDITA_TEST_UMASK" , mask )
f := newProcessFixture ( t , "normal" , "[]" )
paths := [ ] string { f . output , filepath . Join ( f . dir , "errors.yml" ) , filepath . Join ( f . dir , "recoveries.yml" ) , filepath . Join ( f . dir , "report.md" ) }
parsed , _ := strconv . ParseUint ( mask , 8 , 32 )
want := os . FileMode ( 0644 &^ parsed )
for attempt := 0 ; attempt < 2 ; attempt ++ {
code , _ , stderr := f . run ( t , true )
if code != 0 {
t . Fatalf ( "run: %d %s" , code , stderr )
}
for _ , path := range paths {
info , err := os . Stat ( path )
if err != nil {
t . Fatal ( err )
}
if info . Mode ( ) . Perm ( ) != want {
t . Errorf ( "%s mode=%o want=%o" , filepath . Base ( path ) , info . Mode ( ) . Perm ( ) , want )
}
if err := os . Chmod ( path , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
want = 0600 // Existing restrictive modes survive replacement under either mask.
}
} )
}
}
func TestSubprocessWhitespaceOutputIsALiteralFile ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[]" )
f . output = " "
code , stdout , stderr := f . run ( t , true )
if code != 0 || stdout != "" {
t . Fatalf ( "code=%d stdout=%q stderr=%q" , code , stdout , stderr )
}
output , err := os . ReadFile ( filepath . Join ( f . dir , " " ) )
if err != nil || ! bytes . Contains ( output , [ ] byte ( "hello world" ) ) {
t . Fatalf ( "missing transcript: %s %v" , output , err )
}
if f . errors ( t ) . Status != reporting . StatusCompleted {
t . Fatal ( "reports did not follow file output" )
}
}
func TestSubprocessPathAnnouncementsAreRedacted ( t * testing . T ) {
for _ , fail := range [ ] bool { false , true } {
t . Run ( fmt . Sprint ( fail ) , func ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[]" )
selected , unused := "selected-path-secret" , "unused-path-secret"
f . provider . secret = selected
t . Setenv ( "AUDITA_UNUSED_PATH_KEY" , unused )
cfg , err := os . ReadFile ( f . config )
if err != nil {
t . Fatal ( err )
}
cfg = bytes . Replace ( cfg , [ ] byte ( "backends:\n" ) , [ ] byte ( "backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n" ) , 1 )
if err := os . WriteFile ( f . config , cfg , 0600 ) ; err != nil {
t . Fatal ( err )
}
reportDir := filepath . Join ( f . dir , unused )
if err := os . Mkdir ( reportDir , 0700 ) ; err != nil {
t . Fatal ( err )
}
f . output = filepath . Join ( reportDir , "out.json" )
debugParent := filepath . Join ( f . dir , selected )
if fail {
if err := os . WriteFile ( f . transcript , [ ] byte ( "invalid" ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
code , _ , stderr := f . run ( t , true , "--debug" , "--work-dir" , debugParent )
if ( ! fail && code != 0 ) || ( fail && code != 1 ) {
t . Fatalf ( "code=%d stderr=%q" , code , stderr )
}
if strings . Contains ( stderr , selected ) || strings . Contains ( stderr , unused ) {
t . Fatalf ( "credential in stderr: %q" , stderr )
}
if ! strings . Contains ( stderr , "[REDACTED]" ) {
t . Fatalf ( "missing redacted announcement: %q" , stderr )
}
if _ , err := os . Stat ( filepath . Join ( reportDir , "report.md" ) ) ; err != nil {
t . Fatal ( err )
}
dirs , err := os . ReadDir ( debugParent )
if err != nil || len ( dirs ) != 1 {
t . Fatalf ( "debug location: %v %v" , dirs , err )
}
if ! fail {
if _ , err := os . Stat ( f . output ) ; err != nil {
t . Fatal ( err )
}
}
} )
}
}
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal ( t * testing . T ) {
f := newProcessFixture ( t , "normal" , "[grammar]" )
code , _ , baselineSummary := f . run ( t , true )
if code != 0 {
t . Fatal ( baselineSummary )
}
baseline , err := os . ReadFile ( f . output )
if err != nil {
t . Fatal ( err )
}
f . provider . mu . Lock ( )
f . provider . beforeResponse = func ( ) {
dirs , err := filepath . Glob ( filepath . Join ( f . dir , "runs" , "run-*" ) )
if err != nil || len ( dirs ) != 1 {
t . Errorf ( "debug directory: %v %v" , dirs , err )
return
}
if err := os . WriteFile ( filepath . Join ( dirs [ 0 ] , "grammar" ) , [ ] byte ( "block interaction writes" ) , 0600 ) ; err != nil {
t . Error ( err )
}
}
f . provider . mu . Unlock ( )
code , _ , stderr := f . run ( t , true , "--debug" )
if code != 0 || ! strings . HasPrefix ( stderr , strings . SplitN ( baselineSummary , "\n" , 2 ) [ 0 ] ) {
t . Fatalf ( "changed outcome: %d %q" , code , stderr )
}
if ! strings . Contains ( stderr , "debug proposal interaction" ) || ! strings . Contains ( stderr , "debug validation interaction" ) {
t . Fatalf ( "missing debug failures: %q" , stderr )
}
output , err := os . ReadFile ( f . output )
if err != nil || ! bytes . Equal ( output , baseline ) {
t . Fatalf ( "changed transcript: %s %v" , output , err )
}
document := f . errors ( t )
if len ( document . Errors ) != 0 || * document . Summary . Recoveries != 0 {
t . Fatalf ( "debug failure classified as model incident: %+v" , document )
}
}
2026-09-25 00:36:44 +00:00
func TestSubprocessInPlaceTranscript ( t * testing . T ) {
for _ , invalid := range [ ] bool { false , true } {
t . Run ( fmt . Sprint ( invalid ) , func ( t * testing . T ) {
f := newProcessFixture ( t , "success" , "[grammar]" )
f . output = f . transcript
if invalid {
if err := os . WriteFile ( f . transcript , [ ] byte ( "invalid" ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
}
before , err := os . ReadFile ( f . transcript )
if err != nil {
t . Fatal ( err )
}
code , stdout , stderr := f . run ( t , true )
after , err := os . ReadFile ( f . transcript )
if err != nil {
t . Fatal ( err )
}
if invalid {
if code != 1 || ! bytes . Equal ( before , after ) {
t . Fatalf ( "failed processing changed input: code=%d %s" , code , stderr )
}
} else if code != 0 || stdout != "" || ! bytes . Contains ( after , [ ] byte ( "Hello world" ) ) {
t . Fatalf ( "in-place publication failed: code=%d %s %s" , code , stderr , after )
}
} )
}
}
func TestSubprocessProfileSources ( t * testing . T ) {
f := newProcessFixture ( t , "success" , "[]" )
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
loaded := filepath . Join ( f . dir , "profiles" , "errors.yml" )
if err := os . Rename ( filepath . Join ( f . dir , "profiles" , "proposal.yaml" ) , loaded ) ; err != nil {
t . Fatal ( err )
}
before , err := os . ReadFile ( loaded )
if err != nil {
t . Fatal ( err )
}
f . output = filepath . Join ( f . dir , "profiles" , "proposal.yml" )
code , _ , stderr := f . run ( t , true , "--reporting=false" )
if code != 0 {
t . Fatalf ( "unused alternate rejected: %d %s" , code , stderr )
}
// Remove the transcript from the profile directory before the next load.
if err := os . Remove ( f . output ) ; err != nil {
t . Fatal ( err )
}
code , _ , stderr = f . run ( t , true )
after , err := os . ReadFile ( loaded )
if err != nil {
t . Fatal ( err )
}
if code != 2 || ! bytes . Equal ( before , after ) || f . provider . callCount ( ) != 0 {
t . Fatalf ( "loaded profile not protected: %d %s" , code , stderr )
}
}