package main import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "os/signal" "path/filepath" "reflect" "strconv" "strings" "sync" "syscall" "testing" "time" "gitea.maximumdirect.net/eric/audita/v3/internal/cli" "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gopkg.in/yaml.v3" ) func TestProcessSubprocessHelper(t *testing.T) { if os.Getenv("AUDITA_HELPER_PROCESS") != "1" { return } for i, arg := range os.Args { if arg == "--" { if value := os.Getenv("AUDITA_TEST_UMASK"); value != "" { mask, err := strconv.ParseUint(value, 8, 32) if err != nil { os.Exit(2) } syscall.Umask(int(mask)) } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr) stop() os.Exit(code) } } os.Exit(2) } type fixtureProvider struct { mu sync.Mutex calls int proposalCalls int validationCalls int validationRequests [][]int extraDecisionSent bool mode string secret string entered chan struct{} beforeResponse func() } func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { var request struct { Messages []struct { Content string `json:"content"` } `json:"messages"` ResponseFormat struct { JSONSchema struct { Schema struct { Properties map[string]any `json:"properties"` } `json:"schema"` } `json:"json_schema"` } `json:"response_format"` } if err := json.NewDecoder(r.Body).Decode(&request); err != nil { http.Error(w, "bad request", 400) return } _, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"] var requestedIndices []int if validation && strings.HasPrefix(p.mode, "decision-") { var err error requestedIndices, err = fixtureValidationIndices(request.Messages) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } p.mu.Lock() p.calls++ if validation { p.validationCalls++ } else { p.proposalCalls++ } validationCall := p.validationCalls proposalCall := p.proposalCalls if validation && strings.HasPrefix(p.mode, "decision-") { p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...)) } addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent if addExtra { p.extraDecisionSent = true } beforeResponse := p.beforeResponse p.mu.Unlock() if beforeResponse != nil { beforeResponse() } if p.mode == "wait-cancel" { select { case p.entered <- struct{}{}: default: } <-r.Context().Done() return } if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) { w.WriteHeader(http.StatusInternalServerError) _, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret) return } content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}` if p.mode == "invalid-credential-echo" { sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret}) content = string(raw) } if p.mode == "repair-filter-later-failure" && !validation { if proposalCall == 1 { content = `{"wrong":true}` } else { content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X"},{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}` } } if strings.HasPrefix(p.mode, "span-") { switch p.mode { case "span-clean", "span-filter": content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"}]}` if p.mode == "span-filter" { content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"},{"id":3,"original_text":"uh","corrected_text":""}]}` } case "span-spacing": content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" "}]}` case "span-negative": content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":""}]}` } if validation { if p.mode == "span-clean" || p.mode == "span-filter" { content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"},{"correction_index":2,"approved":true,"reason":"ok"}]}` } else { content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}` } if p.mode == "span-negative" && validationCall == 2 { content = `{"validations":[{"correction_index":1,"approved":false,"reason":"would reverse meaning"}]}` } } } if strings.HasPrefix(p.mode, "decision-") && !validation { content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"}]}` if p.mode == "decision-mixed" { content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"},{"id":2,"original_text":"cat","corrected_text":"dog"},{"id":3,"original_text":"bad","corrected_text":"good"},{"id":4,"original_text":"slow","corrected_text":"fast"}]}` } } if validation && !strings.HasPrefix(p.mode, "span-") { content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}` if p.mode == "invalid-decision-after-repair" { if validationCall == 1 { content = `{"wrong":true}` } else { content = `{"validations":[{"correction_index":9,"approved":true,"reason":"ok"}]}` } } } if validation && strings.HasPrefix(p.mode, "decision-") { decisions := make([]map[string]any, 0, len(requestedIndices)+2) for _, index := range requestedIndices { if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 { continue } approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2) decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "reason": "controlled review"}) if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 { decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "reason": "conflicting review"}) } } if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra { decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "reason": "unexpected review"}) } raw, _ := json.Marshal(map[string]any{"validations": decisions}) content = string(raw) } encoded, _ := json.Marshal(content) w.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) } // Read the supplied candidate JSON independently of the bundled instruction prose. type fixtureValidationItem struct { CorrectionIndex int `json:"correction_index"` OriginalText string `json:"original_text"` } func fixtureValidationItems(content string) ([]fixtureValidationItem, bool) { for offset := 0; offset < len(content); offset++ { if content[offset] != '[' { continue } var items []fixtureValidationItem if err := json.Unmarshal([]byte(content[offset:]), &items); err == nil && len(items) > 0 && items[0].CorrectionIndex > 0 { return items, true } } return nil, false } func fixtureValidationIndices(messages []struct { Content string `json:"content"` }) ([]int, error) { for _, message := range messages { if items, ok := fixtureValidationItems(message.Content); ok { indices := make([]int, len(items)) for i, item := range items { indices[i] = item.CorrectionIndex } return indices, nil } } return nil, fmt.Errorf("validation request items not found") } func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls } type processFixture struct { dir string config string transcript string glossary string output string provider *fixtureProvider server *httptest.Server } func newProcessFixture(t *testing.T, mode, modules string) processFixture { t.Helper() dir := t.TempDir() p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)} if mode == "invalid-credential-echo" { p.secret = " synthetic-echo-credential\t " } server := httptest.NewServer(http.HandlerFunc(p.serve)) t.Cleanup(server.Close) resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL) if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil { t.Fatal(err) } configPath := filepath.Join(dir, "config.yml") repairs := 0 if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" { repairs = 1 } if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { repairs = 1 } var stageList []string for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") { if name = strings.TrimSpace(name); name != "" { stageList = append(stageList, " - module: "+name) } } pipeline := " stages: []\n" if len(stageList) != 0 { pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n" } configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n correction_repair_attempts: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs")) if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" } if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil { t.Fatal(err) } transcript := filepath.Join(dir, "input.json") if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil { t.Fatal(err) } glossary := filepath.Join(dir, "glossary.yaml") if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil { t.Fatal(err) } return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary, output: filepath.Join(dir, "output.json"), provider: p, server: server} } func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) { t.Helper() args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config} if includeOutput { args = append(args, "--output", f.output) } args = append(args, extra...) command := exec.Command(os.Args[0], args...) command.Dir = f.dir command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr err := command.Run() code := 0 if err != nil { if exit, ok := err.(*exec.ExitError); ok { code = exit.ExitCode() } else { t.Fatal(err) } } return code, stdout.String(), stderr.String() } func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) { t.Helper() command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view) command.Dir = f.dir command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr err := command.Run() code := 0 if err != nil { if exit, ok := err.(*exec.ExitError); ok { code = exit.ExitCode() } else { t.Fatal(err) } } return code, stdout.String(), stderr.String() } func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument { t.Helper() raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml")) if err != nil { t.Fatal(err) } var document reporting.ErrorsDocument if err := yaml.Unmarshal(raw, &document); err != nil { t.Fatal(err) } return document } func (f processFixture) recoveries(t *testing.T) reporting.RecoveriesDocument { t.Helper() raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")) if err != nil { t.Fatal(err) } var document reporting.RecoveriesDocument if err := yaml.Unmarshal(raw, &document); err != nil { t.Fatal(err) } return document } func (f processFixture) setSegments(t *testing.T, texts ...string) { t.Helper() segments := make([]schema.Segment, len(texts)) for i, value := range texts { segments[i] = schema.Segment{ID: i + 1, Speaker: "A", Start: float64(i * 10), End: float64(i*10 + 1), Text: value, Categories: []string{"dialogue"}} } raw, err := json.Marshal(segments) if err != nil { t.Fatal(err) } if err := os.WriteFile(f.transcript, raw, 0600); err != nil { t.Fatal(err) } } func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transcript { t.Helper() raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } transcript, err := schema.ParseTranscriptJSON(raw) if err != nil { t.Fatal(err) } return transcript } func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) { t.Helper() dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) if err != nil || len(dirs) != 1 { t.Fatalf("expected one debug run: %v %v", dirs, err) } normalized := f.readTranscript(t, filepath.Join(dirs[0], "normalized-transcript.json")) raw, err := os.ReadFile(filepath.Join(dirs[0], "correction-ledger.json")) if err != nil { t.Fatal(err) } var ledger []map[string]json.RawMessage if err := json.Unmarshal(raw, &ledger); err != nil { t.Fatal(err) } return normalized, ledger } func assertSpanMetadata(t *testing.T, normalized, output *schema.Transcript) { t.Helper() if len(normalized.Segments) != len(output.Segments) { t.Fatalf("segment membership changed: normalized=%+v output=%+v", normalized, output) } for i := range normalized.Segments { normalized.Segments[i].Text = output.Segments[i].Text } if !reflect.DeepEqual(normalized, output) { t.Fatalf("segment metadata/order changed: normalized=%+v output=%+v", normalized, output) } } func ledgerDisposition(entry map[string]json.RawMessage) string { var value string _ = json.Unmarshal(entry["disposition"], &value) return value } func ledgerHasDecision(t *testing.T, entry map[string]json.RawMessage, field, key string, approved bool) bool { t.Helper() var decisions []struct { ValidatorKey string `json:"validator_key"` Approved bool `json:"approved"` } if err := json.Unmarshal(entry[field], &decisions); err != nil { t.Fatal(err) } for _, decision := range decisions { if decision.ValidatorKey == key && decision.Approved == approved { return true } } return false } func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) { t.Helper() dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) if err != nil || len(dirs) != 1 { t.Fatalf("debug run: %v %v", dirs, err) } raw, err := os.ReadFile(filepath.Join(dirs[0], "llm-execution.json")) if err != nil { t.Fatal(err) } var execution struct { Calls []struct { Summary struct { ProviderAttempts int `json:"provider_attempts"` StructuralRepairs int `json:"structural_repairs"` } `json:"summary"` } `json:"calls"` } if err := json.Unmarshal(raw, &execution); err != nil { t.Fatal(err) } if len(execution.Calls) != wantCalls || f.provider.callCount() != wantCalls { t.Fatalf("unexpected generation counts: calls=%d provider=%d", len(execution.Calls), f.provider.callCount()) } for _, call := range execution.Calls { if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 { t.Fatalf("unexpected structural repair: %+v", call) } } } func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage { t.Helper() paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json")) if err != nil { t.Fatal(err) } var diagnostics []map[string]json.RawMessage for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var payload map[string]json.RawMessage if err := json.Unmarshal(raw, &payload); err != nil { t.Fatal(err) } if _, validation := payload["reconciliation"]; validation { diagnostics = append(diagnostics, payload) } } return diagnostics } func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int { t.Helper() f.provider.mu.Lock() requests := append([][]int(nil), f.provider.validationRequests...) providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls f.provider.mu.Unlock() if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations { t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests) } paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json")) if err != nil || len(paths) != 1 { t.Fatalf("execution path: %v %v", paths, err) } raw, err := os.ReadFile(paths[0]) if err != nil { t.Fatal(err) } var execution struct { Calls []struct { Summary struct { ProviderAttempts int `json:"provider_attempts"` StructuralRepairs int `json:"structural_repairs"` } `json:"summary"` } `json:"calls"` } if err := json.Unmarshal(raw, &execution); err != nil { t.Fatal(err) } if len(execution.Calls) != providerCalls { t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls) } for _, call := range execution.Calls { if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 { t.Fatalf("extra generation or structural repair: %+v", call) } } metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json")) if err != nil { t.Fatal(err) } validatorsSeen := make(map[string]bool) for _, path := range metadataPaths { raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } var metadata struct { ValidatorName string `json:"validator_name"` BatchIndex int `json:"batch_index"` } if err := json.Unmarshal(raw, &metadata); err != nil { t.Fatal(err) } if metadata.ValidatorName == "" { continue } if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 { t.Fatalf("duplicate or split validation call: %+v", metadata) } validatorsSeen[metadata.ValidatorName] = true } if len(validatorsSeen) != validations { t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations) } return requests } func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) { f := newProcessFixture(t, "decision-mixed", "[grammar]") f.setSegments(t, "teh", "cat", "bad", "slow") code, stdout, stderr := f.run(t, true, "--debug") if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output := f.readTranscript(t, f.output) normalized, ledger := f.debugArtifacts(t) assertSpanMetadata(t, normalized, output) if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" { t.Fatalf("partial validation transcript: %+v", output) } errors, recoveries := f.errors(t), f.recoveries(t) if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 { t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries) } if len(ledger) != 4 { t.Fatalf("unexpected candidate ledger entries: %+v", ledger) } for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} { entry := ledger[index] var candidate int var reason string if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil { t.Fatal(err) } if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 { t.Fatal(err) } if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) { t.Fatalf("candidate %d evidence: %+v", index, entry) } if index > 0 { var decisions []struct { Disposition string `json:"disposition"` } if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition { t.Fatalf("candidate %d decision: %+v %v", index, decisions, err) } } } var mixedSeen bool for _, diagnostic := range f.validationDiagnostics(t) { var reconciliation struct { UnavailableIndices []int `json:"unavailable_indices"` IgnoredIndices []int `json:"ignored_indices"` Decisions []struct { ProposalIndex int `json:"proposal_index"` ReasonCode string `json:"reason_code"` } `json:"decisions"` } if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil { t.Fatal(err) } if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) { continue } var decoded struct { Validations []struct { CorrectionIndex int `json:"correction_index"` } `json:"validations"` } if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil { t.Fatal(err) } var finalContent string if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil { t.Fatal(err) } if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" { t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded) } mixedSeen = true } if !mixedSeen { t.Fatal("missing mixed decision interaction evidence") } requests := f.assertDecisionRequestsUseOneAttempt(t) full, survivor := 0, 0 for _, indices := range requests { switch { case reflect.DeepEqual(indices, []int{1, 2, 3, 4}): full++ case reflect.DeepEqual(indices, []int{1}): survivor++ default: t.Fatalf("unexpected validation identities: %v", requests) } } if full != 1 || survivor == 0 { t.Fatalf("wrong validation progression: %v", requests) } } func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) { f := newProcessFixture(t, "decision-extra-only", "[grammar]") f.setSegments(t, "teh") code, stdout, stderr := f.run(t, true, "--debug") if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output := f.readTranscript(t, f.output) if len(output.Segments) != 1 || output.Segments[0].Text != "the" { t.Fatalf("lost usable correction: %+v", output) } errors, recoveries := f.errors(t), f.recoveries(t) if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 { t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries) } markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md")) if err != nil { t.Fatal(err) } if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) { t.Fatalf("Markdown totals disagree: %s", markdown) } _, ledger := f.debugArtifacts(t) if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" { t.Fatalf("unknown candidate in ledger: %+v", ledger) } ignored := 0 for _, diagnostic := range f.validationDiagnostics(t) { var reconciliation struct { IgnoredIndices []int `json:"ignored_indices"` } if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil { t.Fatal(err) } if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) { ignored++ } } if ignored != 1 { t.Fatalf("expected one handled extra decision, got %d", ignored) } requests := f.assertDecisionRequestsUseOneAttempt(t) for _, indices := range requests { if !reflect.DeepEqual(indices, []int{1}) { t.Fatalf("unexpected identities: %v", requests) } } } func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) { f := newProcessFixture(t, "span-clean", "[spoken_word]") f.setSegments(t, "uh hello", "I I think") code, stdout, stderr := f.run(t, true, "--debug") if code != 0 || stdout != "" || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 0 recoveries") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output := f.readTranscript(t, f.output) normalized, ledger := f.debugArtifacts(t) assertSpanMetadata(t, normalized, output) if len(output.Segments) != 2 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" { t.Fatalf("unexpected corrected transcript: %+v", output) } if len(ledger) != 2 || ledgerDisposition(ledger[0]) != "applied" || ledgerDisposition(ledger[1]) != "applied" || string(ledger[0]["proposed_corrected_text"]) != `""` || string(ledger[0]["applied_corrected_text"]) != `""` { t.Fatalf("deletion missing from ledger: %+v", ledger) } errors, recoveries := f.errors(t), f.recoveries(t) if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 0 || *errors.Summary.CorrectionsApplied != 2 || *errors.Summary.WorkUnitsCompleted != 1 || *recoveries.Summary.CorrectionsApplied != 2 { t.Fatalf("report totals disagree: errors=%+v recoveries=%+v", errors, recoveries) } f.assertNoStructuralRepairs(t, 3) } func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) { f := newProcessFixture(t, "span-spacing", "[grammar]") f.setSegments(t, "hello world") code, _, stderr := f.run(t, true, "--debug") if code != 0 || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") { t.Fatalf("code=%d stderr=%q", code, stderr) } output := f.readTranscript(t, f.output) normalized, _ := f.debugArtifacts(t) assertSpanMetadata(t, normalized, output) if len(output.Segments) != 1 || output.Segments[0].Text != "hello world" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 { t.Fatalf("spacing correction failed: %+v", output) } f.assertNoStructuralRepairs(t, 3) } func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) { f := newProcessFixture(t, "span-filter", "[spoken_word]") f.setSegments(t, "uh hello", "I I think", "uh") code, _, stderr := f.run(t, true, "--debug") if code != 0 || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 1 recoveries") { t.Fatalf("code=%d stderr=%q", code, stderr) } output := f.readTranscript(t, f.output) normalized, ledger := f.debugArtifacts(t) assertSpanMetadata(t, normalized, output) if len(output.Segments) != 3 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" || output.Segments[2].Text != "uh" { t.Fatalf("filtered segment or survivors wrong: %+v", output) } recoveries := f.recoveries(t) if len(f.errors(t).Errors) != 0 || len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].Reasons) != 1 || recoveries.Recoveries[0].Reasons[0] != "proposals_filtered" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || recoveries.Recoveries[0].FilteredProposals[0].Reason != "empty_resulting_segment" || *recoveries.Summary.ProposalResponsesWithFiltering != 1 { t.Fatalf("filtering recovery wrong: %+v", recoveries) } if len(ledger) != 3 || ledgerDisposition(ledger[2]) != "rejected" || string(ledger[2]["proposed_corrected_text"]) != `""` { t.Fatalf("filtered deletion evidence missing: %+v", ledger) } if !ledgerHasDecision(t, ledger[2], "deterministic_validator_decisions", "non_empty_resulting_segment", false) { t.Fatalf("renamed validator missing from evidence: %+v", ledger[2]) } f.assertNoStructuralRepairs(t, 3) } func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) { f := newProcessFixture(t, "span-negative", "[grammar]") f.setSegments(t, "not good") code, _, stderr := f.run(t, true, "--debug") if code != 0 || !strings.Contains(stderr, "0 corrections applied; 0 work units with errors; 0 recoveries") { t.Fatalf("code=%d stderr=%q", code, stderr) } output := f.readTranscript(t, f.output) normalized, ledger := f.debugArtifacts(t) assertSpanMetadata(t, normalized, output) if len(output.Segments) != 1 || output.Segments[0].Text != "not good" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 { t.Fatalf("negative review changed outcome: %+v", output) } if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "rejected" || string(ledger[0]["proposed_corrected_text"]) != `""` { t.Fatalf("rejection evidence missing: %+v", ledger) } if _, present := ledger[0]["applied_corrected_text"]; present || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "editorial_review", true) || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "meaning_reversal_review", false) { t.Fatalf("negative decision not recorded: %+v", ledger[0]) } f.assertNoStructuralRepairs(t, 3) } func TestSubprocessDefaultFileOutputAndReports(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, stdout, stderr := f.run(t, true) if code != 0 || stdout != "" || stderr != "" { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("Hello world")) { t.Fatalf("output: %s %v", output, err) } document := f.errors(t) if document.Status != reporting.StatusCompleted || document.Summary.WorkUnitsCompleted == nil || *document.Summary.WorkUnitsCompleted != 1 || len(document.Errors) != 0 { t.Fatalf("errors document: %+v", document) } for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { report, err := os.ReadFile(filepath.Join(f.dir, name)) if err != nil { t.Fatal(err) } if bytes.Contains(report, []byte("hello world")) || bytes.Contains(report, []byte("Hello world")) || bytes.Contains(report, []byte(f.provider.secret)) { t.Fatalf("ordinary report contains payload or secret: %s", name) } } if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) { t.Fatalf("debug created by default: %v", err) } } func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") reports := filepath.Join(f.dir, "reports") if err := os.Mkdir(reports, 0700); err != nil { t.Fatal(err) } code, stdout, stderr := f.run(t, false, "--report-dir", reports) if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } if _, err := os.Stat(f.output); !os.IsNotExist(err) { t.Fatalf("stdout run wrote transcript file: %v", err) } if _, err := os.Stat(filepath.Join(reports, "errors.yml")); err != nil { t.Fatal(err) } } func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") transcriptDir := filepath.Join(f.dir, "transcripts") reportDir := filepath.Join(f.dir, "reports") debugDir := filepath.Join(f.dir, "debug") for _, dir := range []string{transcriptDir, reportDir} { if err := os.Mkdir(dir, 0700); err != nil { t.Fatal(err) } } f.output = filepath.Join(transcriptDir, "corrected.json") code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir) if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) { t.Fatalf("transcript: %s %v", output, err) } for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { if _, err := os.Stat(filepath.Join(reportDir, name)); err != nil { t.Fatalf("report %s: %v", name, err) } if _, err := os.Stat(filepath.Join(transcriptDir, name)); !os.IsNotExist(err) { t.Fatalf("report %s appeared beside transcript: %v", name, err) } } dirs, err := filepath.Glob(filepath.Join(debugDir, "run-*")) if err != nil || len(dirs) != 1 { t.Fatalf("debug runs: %v %v", dirs, err) } if _, err := os.Stat(filepath.Join(dirs[0], "correction-ledger.json")); err != nil { t.Fatalf("debug capture: %v", err) } } func TestSubprocessReportingAndDebugSelections(t *testing.T) { var expectedOutput, expectedSummary string for _, selection := range []struct{ reporting, debug bool }{ {false, false}, {false, true}, {true, false}, {true, true}, } { f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") reportDir := filepath.Join(f.dir, "missing") extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir} if selection.reporting { reportDir = filepath.Join(f.dir, "selected-reports") if err := os.Mkdir(reportDir, 0700); err != nil { t.Fatal(err) } extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir} } if selection.debug { extra = append(extra, "--debug") } code, stdout, stderr := f.run(t, true, extra...) if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) } output, err := os.ReadFile(f.output) if err != nil { t.Fatal(err) } summary := "" for _, line := range strings.Split(stderr, "\n") { if strings.HasPrefix(line, "Completed:") { summary = line break } } if expectedOutput == "" { expectedOutput, expectedSummary = string(output), summary } else if string(output) != expectedOutput || summary != expectedSummary { t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output) } for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { _, err := os.Stat(filepath.Join(reportDir, name)) if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) { t.Fatalf("selection=%+v report %s: %v", selection, name, err) } } dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) if err != nil || len(dirs) != boolInt(selection.debug) { t.Fatalf("selection=%+v dirs=%v %v", selection, dirs, err) } if selection.debug { for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} { if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil { t.Fatal(err) } } } } } func boolInt(value bool) int { if value { return 1 } return 0 } func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]") code, stdout, stderr := f.run(t, true, "--verbose") if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("hello world")) { t.Fatalf("normalized output: %s %v", output, err) } document := f.errors(t) if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 { t.Fatalf("model failure report: %+v", document) } if strings.Contains(stderr, f.provider.secret) { t.Fatal("secret leaked in stderr") } } func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) { f := newProcessFixture(t, "invalid-credential-echo", "[grammar]") effective := strings.TrimSpace(f.provider.secret) code, stdout, stderr := f.run(t, true, "--debug") if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") { t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr) } if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) { t.Fatal("credential escaped redaction in command output") } responseCaptured := false err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error { if err != nil || entry.IsDir() { return err } data, err := os.ReadFile(path) if err != nil { return err } if bytes.Contains(data, []byte(effective)) { t.Errorf("credential escaped redaction in %s", entry.Name()) } if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) { responseCaptured = true } return nil }) if err != nil || !responseCaptured { t.Fatalf("missing redacted provider echo: %v", err) } } func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) { f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]") code, stdout, stderr := f.run(t, true, "--verbose") if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) { t.Fatalf("unchecked output: %s %v", output, err) } document := f.errors(t) if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 4 { t.Fatalf("validation failure report: %+v; calls=%d", document, f.provider.callCount()) } } func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) { f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t)) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("Hello world")) { t.Fatalf("output: %s %v", output, err) } errors := f.errors(t) if len(errors.Errors) != 1 || errors.Errors[0].ModuleInstance != "grammar_2" { t.Fatalf("later failure: %+v", errors) } raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")) if err != nil { t.Fatal(err) } var recoveries reporting.RecoveriesDocument if err := yaml.Unmarshal(raw, &recoveries); err != nil { t.Fatal(err) } if len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].ModuleInstance != "grammar_1" || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || *recoveries.Summary.Recoveries != 1 { t.Fatalf("combined recovery: %+v", recoveries) } if !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "output_repaired") || !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "proposals_filtered") { t.Fatalf("recovery reasons: %+v", recoveries.Recoveries[0]) } } func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) { f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]") code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") { t.Fatalf("code=%d stderr=%q", code, stderr) } document := f.errors(t) if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || len(document.Errors[0].Failures) != 1 || document.Errors[0].Failures[0].Reason != "validation_decisions_invalid" { t.Fatalf("invalid decisions report: %+v", document) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) { t.Fatalf("unchecked output: %s %v", output, err) } } func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T) { f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]") ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) defer stop() args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output} command := exec.CommandContext(ctx, os.Args[0], args...) command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) var stderr bytes.Buffer command.Stderr = &stderr if err := command.Start(); err != nil { t.Fatal(err) } select { case <-f.provider.entered: case <-ctx.Done(): t.Fatalf("provider was not reached: %v", ctx.Err()) } if err := command.Process.Signal(os.Interrupt); err != nil { t.Fatal(err) } err := command.Wait() if ctx.Err() != nil || err == nil { t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String()) } if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 { t.Fatalf("cancellation report: %+v", document) } if _, err := os.Stat(f.output); !os.IsNotExist(err) { t.Fatalf("canceled run published transcript: %v", err) } } func TestSubprocessPartialValidationAndLaterStage(t *testing.T) { f := newProcessFixture(t, "partial-validation", "[grammar, grammar]") code, _, stderr := f.run(t, true, "--verbose") if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") { t.Fatalf("code=%d stderr=%q", code, stderr) } document := f.errors(t) if len(document.Errors) != 1 || document.Errors[0].ModuleInstance != "grammar_1" || *document.Summary.WorkUnitsCompleted != 2 { t.Fatalf("partial outcome: %+v", document) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Contains(output, []byte("Hello world")) { t.Fatalf("later correction: %s %v", output, err) } } func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil { t.Fatal(err) } code, _, _ := f.run(t, true) if code != 2 || f.provider.callCount() != 0 { t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount()) } if _, err := os.Stat(f.output); !os.IsNotExist(err) { t.Fatalf("failed transcript published: %v", err) } } func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml")) if err != nil { t.Fatal(err) } updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1) if updated == string(original) { t.Fatal("process fixture backend configuration was not found") } if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil { t.Fatal(err) } code, stdout, stderr := f.run(t, true) if code != 2 || stdout != "" || f.provider.callCount() != 0 { t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount()) } } func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil { t.Fatal(err) } reportDir := filepath.Join(f.dir, "failed-reports") if err := os.Mkdir(reportDir, 0700); err != nil { t.Fatal(err) } code, _, _ := f.run(t, true, "--report-dir", reportDir) if code != 2 || f.provider.callCount() != 0 { t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount()) } for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) { t.Fatalf("report created before preflight: %s %v", name, err) } } if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) { t.Fatalf("failed report appeared in default directory: %v", err) } if _, err := os.Stat(f.output); !os.IsNotExist(err) { t.Fatalf("failed transcript published: %v", err) } } func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") f.output = filepath.Join(f.dir, "missing", "output.json") code, _, stderr := f.run(t, true) if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "report directory") { t.Fatalf("preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr) } if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) { t.Fatalf("report initialized before preflight: %v", err) } } func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript) if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") { t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr) } if _, err := os.Stat(f.output); err != nil { t.Fatal(err) } if document := f.errors(t); document.Status != reporting.StatusCompleted { t.Fatalf("report status: %+v", document) } } func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") f.output = filepath.Join(f.dir, "missing", "output.json") code, _, stderr := f.run(t, true, "--reporting=false") if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") { t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr) } } func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) { for _, mask := range []string{"0077", "0022"} { t.Run(mask, func(t *testing.T) { t.Setenv("AUDITA_TEST_UMASK", mask) f := newProcessFixture(t, "normal", "[]") paths := []string{f.output, filepath.Join(f.dir, "errors.yml"), filepath.Join(f.dir, "recoveries.yml"), filepath.Join(f.dir, "report.md")} parsed, _ := strconv.ParseUint(mask, 8, 32) want := os.FileMode(0644 &^ parsed) for attempt := 0; attempt < 2; attempt++ { code, _, stderr := f.run(t, true) if code != 0 { t.Fatalf("run: %d %s", code, stderr) } for _, path := range paths { info, err := os.Stat(path) if err != nil { t.Fatal(err) } if info.Mode().Perm() != want { t.Errorf("%s mode=%o want=%o", filepath.Base(path), info.Mode().Perm(), want) } if err := os.Chmod(path, 0600); err != nil { t.Fatal(err) } } want = 0600 // Existing restrictive modes survive replacement under either mask. } }) } } func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) { f := newProcessFixture(t, "normal", "[]") f.output = " " code, stdout, stderr := f.run(t, true) if code != 0 || stdout != "" { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(filepath.Join(f.dir, " ")) if err != nil || !bytes.Contains(output, []byte("hello world")) { t.Fatalf("missing transcript: %s %v", output, err) } if f.errors(t).Status != reporting.StatusCompleted { t.Fatal("reports did not follow file output") } } func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { for _, fail := range []bool{false, true} { t.Run(fmt.Sprint(fail), func(t *testing.T) { f := newProcessFixture(t, "normal", "[]") selected, unused := "selected-path-secret", "unused-path-secret" f.provider.secret = selected t.Setenv("AUDITA_UNUSED_PATH_KEY", unused) cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml")) if err != nil { t.Fatal(err) } cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1) if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil { t.Fatal(err) } reportDir := filepath.Join(f.dir, unused) if err := os.Mkdir(reportDir, 0700); err != nil { t.Fatal(err) } f.output = filepath.Join(reportDir, "out.json") debugParent := filepath.Join(f.dir, selected) if fail { if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil { t.Fatal(err) } } code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent) if (!fail && code != 0) || (fail && code != 2) { t.Fatalf("code=%d stderr=%q", code, stderr) } if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) { t.Fatalf("credential in stderr: %q", stderr) } if !fail && !strings.Contains(stderr, "[REDACTED]") { t.Fatalf("missing redacted announcement: %q", stderr) } if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil { t.Fatal(err) } dirs, err := os.ReadDir(debugParent) if !fail && (err != nil || len(dirs) != 1) { t.Fatalf("debug location: %v %v", dirs, err) } if !fail { if _, err := os.Stat(f.output); err != nil { t.Fatal(err) } } }) } } func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, _, baselineSummary := f.run(t, true, "--verbose") if code != 0 { t.Fatal(baselineSummary) } baseline, err := os.ReadFile(f.output) if err != nil { t.Fatal(err) } f.provider.mu.Lock() f.provider.beforeResponse = func() { dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) if err != nil || len(dirs) != 1 { t.Errorf("debug directory: %v %v", dirs, err) return } if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil { t.Error(err) } } f.provider.mu.Unlock() code, _, stderr := f.run(t, true, "--debug") if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) { t.Fatalf("changed outcome: %d %q", code, stderr) } if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") { t.Fatalf("missing debug failures: %q", stderr) } output, err := os.ReadFile(f.output) if err != nil || !bytes.Equal(output, baseline) { t.Fatalf("changed transcript: %s %v", output, err) } document := f.errors(t) if len(document.Errors) != 0 || *document.Summary.Recoveries != 0 { t.Fatalf("debug failure classified as model incident: %+v", document) } } func TestSubprocessInPlaceTranscript(t *testing.T) { for _, invalid := range []bool{false, true} { t.Run(fmt.Sprint(invalid), func(t *testing.T) { f := newProcessFixture(t, "success", "[grammar]") f.output = f.transcript if invalid { if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil { t.Fatal(err) } } before, err := os.ReadFile(f.transcript) if err != nil { t.Fatal(err) } code, stdout, stderr := f.run(t, true) after, err := os.ReadFile(f.transcript) if err != nil { t.Fatal(err) } if invalid { if code != 2 || !bytes.Equal(before, after) { t.Fatalf("failed processing changed input: code=%d %s", code, stderr) } } else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) { t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after) } }) } } func TestSubprocessProfileSources(t *testing.T) { f := newProcessFixture(t, "success", "[]") // PromptKit uses profile IDs, not filename stems. A report-named profile is // still an input and must not be overwritten during report initialization. if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil { t.Fatal(err) } loaded := filepath.Join(f.dir, "prompts", "errors.yml") if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil { t.Fatal(err) } before, err := os.ReadFile(loaded) if err != nil { t.Fatal(err) } f.output = filepath.Join(f.dir, "prompts", "proposal.yml") code, _, stderr := f.run(t, true, "--reporting=false") if code != 0 { t.Fatalf("unused alternate rejected: %d %s", code, stderr) } // Remove the transcript from the profile directory before the next load. if err := os.Remove(f.output); err != nil { t.Fatal(err) } code, _, stderr = f.run(t, true) after, err := os.ReadFile(loaded) if err != nil { t.Fatal(err) } if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 { t.Fatalf("loaded profile not protected: %d %s", code, stderr) } parentDir := filepath.Join(f.dir, "prompts", "catalog") childDir := filepath.Join(f.dir, "prompts", "nested", "child") for _, directory := range []string{parentDir, childDir} { if err := os.MkdirAll(directory, 0700); err != nil { t.Fatal(err) } } parent := filepath.Join(parentDir, "arbitrary-name.yaml") child := filepath.Join(childDir, "override.yaml") resource := filepath.Join(parentDir, "report.md") for path, content := range map[string]string{ parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n", child: "prompts:\n - id: child\n extends: {id: source}\n", resource: "Captured operator message.\n", } { if err := os.WriteFile(path, []byte(content), 0600); err != nil { t.Fatal(err) } } resourceBefore, err := os.ReadFile(resource) if err != nil { t.Fatal(err) } f.output = resource code, _, stderr = f.run(t, true, "--reporting=false") resourceAfter, err := os.ReadFile(resource) if err != nil { t.Fatal(err) } if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr) } f.output = filepath.Join(f.dir, "output.json") code, _, stderr = f.run(t, true, "--report-dir", parentDir) resourceAfter, err = os.ReadFile(resource) if err != nil { t.Fatal(err) } if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 { t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr) } }