eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
cmd/audita/main_integration_test.go
View File

@@ -233,15 +233,10 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
for _, id := range []string{"proposal", "validation"} {
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
}
configPath := filepath.Join(dir, "config.yml")
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
@@ -250,7 +245,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
var stageList []string
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
if name = strings.TrimSpace(name); name != "" {
stageList = append(stageList, " - module: "+name)
}
}
pipeline := " stages: []\n"
if len(stageList) != 0 {
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
}
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
@@ -271,7 +276,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
t.Helper()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
if includeOutput {
args = append(args, "--output", f.output)
}
@@ -775,7 +780,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
if code != 0 || stdout != "" || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
@@ -807,7 +812,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
@@ -830,7 +835,7 @@ func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testin
}
f.output = filepath.Join(transcriptDir, "corrected.json")
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
@@ -859,42 +864,34 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
{false, false}, {false, true}, {true, false}, {true, true},
} {
f := newProcessFixture(t, "normal", "[grammar]")
if selection.reporting && selection.debug {
original, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
updated := strings.Replace(string(original), old, inherited, 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
t.Fatal(err)
}
}
reportDir := filepath.Join(f.dir, "missing")
extra := []string{"--reporting=false", "--report-dir", reportDir}
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
if selection.reporting {
reportDir = filepath.Join(f.dir, "selected-reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
extra = []string{"--reporting=true", "--report-dir", reportDir}
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
}
if selection.debug {
extra = append(extra, "--debug")
}
code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
summary := strings.SplitN(stderr, "\n", 2)[0]
summary := ""
for _, line := range strings.Split(stderr, "\n") {
if strings.HasPrefix(line, "Completed:") {
summary = line
break
}
}
if expectedOutput == "" {
expectedOutput, expectedSummary = string(output), summary
} else if string(output) != expectedOutput || summary != expectedSummary {
@@ -916,31 +913,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
t.Fatal(err)
}
}
if selection.reporting {
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
t.Fatalf("effective inherited settings: %s", effective)
}
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
if !strings.Contains(string(origins), want) {
t.Fatalf("configuration origins missing %q: %s", want, origins)
}
}
if strings.Contains(string(origins), f.provider.secret) {
t.Fatal("configuration origins exposed the resolved inherited credential")
}
code, explain, stderr := f.runConfigView(t, "explain")
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
}
}
}
}
}
@@ -954,7 +927,7 @@ func boolInt(value bool) int {
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -973,7 +946,7 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -989,7 +962,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
}
@@ -1019,7 +992,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1037,7 +1010,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stderr bytes.Buffer
@@ -1067,7 +1040,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1081,55 +1054,41 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
}
}
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
}
code, _, _ := f.run(t, true)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failed transcript published: %v", err)
}
var recoveries reporting.RecoveriesDocument
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
t.Fatal(err)
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
t.Fatal(err)
}
if recoveries.Status != reporting.StatusFailed {
t.Fatalf("recoveries status: %+v", recoveries)
}
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
}
}
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
original, err := os.ReadFile(f.config)
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
}
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
@@ -1139,25 +1098,13 @@ func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing
t.Fatal(err)
}
code, _, _ := f.run(t, true, "--report-dir", reportDir)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
if err != nil {
t.Fatal(err)
}
var document reporting.ErrorsDocument
if err := yaml.Unmarshal(raw, &document); err != nil {
t.Fatal(err)
}
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
t.Fatal(err)
}
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
t.Fatalf("report created before preflight: %s %v", name, err)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("failed report appeared in default directory: %v", err)
@@ -1182,7 +1129,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
}
if _, err := os.Stat(f.output); err != nil {
@@ -1197,7 +1144,7 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
f.output = filepath.Join(f.dir, "missing", "output.json")
code, _, stderr := f.run(t, true, "--reporting=false")
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
}
@@ -1256,12 +1203,12 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
selected, unused := "selected-path-secret", "unused-path-secret"
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(f.config)
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
@@ -1276,20 +1223,20 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
}
}
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
if (!fail && code != 0) || (fail && code != 1) {
if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
t.Fatalf("credential in stderr: %q", stderr)
}
if !strings.Contains(stderr, "[REDACTED]") {
if !fail && !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err)
}
dirs, err := os.ReadDir(debugParent)
if err != nil || len(dirs) != 1 {
if !fail && (err != nil || len(dirs) != 1) {
t.Fatalf("debug location: %v %v", dirs, err)
}
if !fail {
@@ -1303,7 +1250,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, baselineSummary := f.run(t, true)
code, _, baselineSummary := f.run(t, true, "--verbose")
if code != 0 {
t.Fatal(baselineSummary)
}
@@ -1318,16 +1265,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
t.Errorf("debug directory: %v %v", dirs, err)
return
}
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
t.Error(err)
}
}
f.provider.mu.Unlock()
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
t.Fatalf("changed outcome: %d %q", code, stderr)
}
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
t.Fatalf("missing debug failures: %q", stderr)
}
output, err := os.ReadFile(f.output)
@@ -1360,7 +1307,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) {
t.Fatal(err)
}
if invalid {
if code != 1 || !bytes.Equal(before, after) {
if code != 2 || !bytes.Equal(before, after) {
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
}
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
@@ -1374,15 +1321,18 @@ func TestSubprocessProfileSources(t *testing.T) {
f := newProcessFixture(t, "success", "[]")
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
t.Fatal(err)
}
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
code, _, stderr := f.run(t, true, "--reporting=false")
if code != 0 {
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
@@ -1396,12 +1346,12 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
}
parentDir := filepath.Join(f.dir, "profiles", "catalog")
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
parentDir := filepath.Join(f.dir, "prompts", "catalog")
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
@@ -1411,8 +1361,8 @@ func TestSubprocessProfileSources(t *testing.T) {
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "report.md")
for path, content := range map[string]string{
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
child: "prompts:\n literal/~child:\n extends: source\n",
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
child: "prompts:\n - id: child\n extends: {id: source}\n",
resource: "Captured operator message.\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
@@ -1429,7 +1379,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
}
@@ -1439,7 +1389,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
}
}