diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 796fbc8..6cd5001 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -118,14 +119,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) { w.WriteHeader(http.StatusInternalServerError) - _, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret) + _, _ = io.WriteString(w, `{"error":{"message":"provider unavailable"}}`) return } content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}` - if p.mode == "invalid-credential-echo" { - sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") - raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret}) - content = string(raw) + if p.mode == "malformed-response" { + content = `{"corrections": [` } if p.mode == "repair-filter-later-failure" && !validation { if proposalCall == 1 { @@ -246,9 +245,6 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { t.Helper() dir := t.TempDir() p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)} - if mode == "invalid-credential-echo" { - p.secret = " synthetic-echo-credential\t " - } server := httptest.NewServer(http.HandlerFunc(p.serve)) t.Cleanup(server.Close) resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL) @@ -957,40 +953,37 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 { t.Fatalf("model failure report: %+v", document) } - if strings.Contains(stderr, f.provider.secret) { - t.Fatal("secret leaked in stderr") - } } -func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) { - f := newProcessFixture(t, "invalid-credential-echo", "[grammar]") - effective := strings.TrimSpace(f.provider.secret) +func TestSubprocessCapturesMalformedProviderResponse(t *testing.T) { + f := newProcessFixture(t, "malformed-response", "[grammar]") code, stdout, stderr := f.run(t, true, "--debug") if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") { t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr) } - if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) { - t.Fatal("credential escaped redaction in command output") - } responseCaptured := false err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error { - if err != nil || entry.IsDir() { + if err != nil { return err } + if entry.IsDir() || !strings.HasSuffix(entry.Name(), "-response-payload.json") { + return nil + } data, err := os.ReadFile(path) if err != nil { return err } - if bytes.Contains(data, []byte(effective)) { - t.Errorf("credential escaped redaction in %s", entry.Name()) + var response struct { + FinalContent string `json:"final_content"` } - if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) { - responseCaptured = true + if err := json.Unmarshal(data, &response); err != nil { + return err } + responseCaptured = responseCaptured || response.FinalContent == `{"corrections": [` return nil }) if err != nil || !responseCaptured { - t.Fatalf("missing redacted provider echo: %v", err) + t.Fatalf("missing malformed provider response: %v", err) } } @@ -1246,27 +1239,16 @@ func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) { } } -func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { +func TestSubprocessAnnouncesActualOutputPaths(t *testing.T) { for _, fail := range []bool{false, true} { t.Run(fmt.Sprint(fail), func(t *testing.T) { f := newProcessFixture(t, "normal", "[]") - selected, unused := "selected-path-secret", "unused-path-secret" - f.provider.secret = selected - t.Setenv("AUDITA_UNUSED_PATH_KEY", unused) - cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml")) - if err != nil { - t.Fatal(err) - } - cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1) - if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil { - t.Fatal(err) - } - reportDir := filepath.Join(f.dir, unused) + reportDir := filepath.Join(f.dir, "reports") if err := os.Mkdir(reportDir, 0700); err != nil { t.Fatal(err) } f.output = filepath.Join(reportDir, "out.json") - debugParent := filepath.Join(f.dir, selected) + debugParent := filepath.Join(f.dir, "diagnostics") if fail { if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil { t.Fatal(err) @@ -1276,11 +1258,8 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { if (!fail && code != 0) || (fail && code != 2) { t.Fatalf("code=%d stderr=%q", code, stderr) } - if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) { - t.Fatalf("credential in stderr: %q", stderr) - } - if !fail && !strings.Contains(stderr, "[REDACTED]") { - t.Fatalf("missing redacted announcement: %q", stderr) + if !fail && (!strings.Contains(stderr, filepath.Join(reportDir, "report.md")) || !strings.Contains(stderr, debugParent)) { + t.Fatalf("missing destination announcement: %q", stderr) } if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil { t.Fatal(err) diff --git a/docs/internal/cli-config.md b/docs/internal/cli-config.md index 62a6f75..1ddd6fd 100644 --- a/docs/internal/cli-config.md +++ b/docs/internal/cli-config.md @@ -11,11 +11,10 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the applicable. Preserve the precedence defined in the [configuration contract](../config.md#loading-and-composition), including the distinct validation and effective-config commands. -3. Update redaction when secrets are involved. -4. Review configuration tests and related CLI tests for precedence, validation, +3. Review configuration tests and related CLI tests for precedence, validation, and source-specific behavior. Add or adjust tests for the changed contract according to the [testing policy](../policy/testing.md). -5. Update the configuration reference and affected [examples](../../examples/). +4. Update the configuration reference and affected [examples](../../examples/). ## Changing CLI Behavior @@ -51,8 +50,7 @@ Select checks according to the [testing policy](../policy/testing.md): - Process/output changes: verify stdout, stderr, exit, output-file, and report behavior against the [CLI contract](../cli.md) and [subprocess guidance](../integrations/subprocess.md), including failures. -- Report or diagnostics changes: review [report assembly tests and redaction - checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths. +- Report or diagnostics changes: review [report assembly tests](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths. - Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests) and `internal/core/schema` tests, including supported formats and rejection of unsupported schemas. diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 2ba98be..674dcfd 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -12,8 +12,7 @@ work-unit errors, separate run errors, and one recovery per successful logical operation. Reports order stages, sections, validators, and batches by semantic identity, never concurrent completion time. The core reporting package owns Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI -completion formatter. YAML redaction walks serialized string nodes so numeric -and boolean values retain their types. Ordinary reports omit raw causes, +completion formatter. Reports encode the typed documents directly to YAML. Ordinary reports omit raw causes, transcript text, complete decisions, and token/timing payloads. Validation operations with unavailable decisions report only affected requested indices, even when other candidates in the batch continue. Ignored extra @@ -34,17 +33,16 @@ pass; a published transcript is never erased by a later report failure. Debug is independent. The shared `--debug` flag enables a unique retained child under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers -when debug is off. With debug enabled, it captures source and normalized -snapshots, invocation, effective settings, correction ledger, LLM execution -observations, and model interactions. Candidate capture includes proposals later +when debug is off. With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized +transcript snapshots. It also captures invocation metadata, effective settings +rendered by ConfigKit, the correction ledger, LLM execution observations, and +model interactions. Candidate capture includes proposals later rejected or interrupted. Repair and revalidation artifacts use section, proposal, -attempt, validator, and batch coordinates as applicable; their retained content -is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied -replacements and distinguishes unavailable candidates from interrupted work. +attempt, validator, and batch coordinates as applicable. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work. A concurrency-safe diagnostic callback reports optional write failures without -aborting processing. Diagnostic redaction uses credential source metadata; debug -artifacts can still contain sensitive transcript and prompt content. Execution +aborting processing. Debug artifacts can contain sensitive transcript and +prompt content. Execution position and optional operator ID are carried separately from generated safe artifact path coordinates. diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index adb48ba..0b33dc0 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -15,9 +15,7 @@ The operation budget includes preparation, admission, retries, and structural repair. `ErrOperationTimeout` remains recoverable even when wrapped by a preparation error. Parent cancellation remains distinct. The prepared notification provides the actual request snapshot; failures before notification have no snapshot. -PromptKit owns backend and credential selection. The adapter uses captured -credential source names for artifact redaction without inspecting unrelated -environment values. See the [integration contract](../integrations/openai-compatible-llm.md) +PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md) for failure behavior and the [configuration reference](../config.md) for sources. When correction repair is enabled, the proposal call captures a native immutable diff --git a/docs/internal/overview.md b/docs/internal/overview.md index ae3a88d..6753b87 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -67,7 +67,7 @@ artifact lifecycle. ## Component Responsibilities -Core components implement configuration defaults/loading/validation/redaction, +Core components implement configuration defaults, loading, and validation, transcript schemas and derived glossary views, normalization and chunking, output encoding, diagnostics artifact helpers, and public report structures. diff --git a/docs/migrations/v2.md b/docs/migrations/v2.md index 16b0ec8..33b6ee8 100644 --- a/docs/migrations/v2.md +++ b/docs/migrations/v2.md @@ -121,8 +121,7 @@ provider attempts. With debug enabled, utilization remains grouped by backend capacity and usage fields retain known/complete flags. Debug runs write `effective-config.redacted.yaml`, `selected-profiles.yaml`, `config-origins.yaml`, `utilization-diagnostics.json`, and -`llm-execution.json`. Request and response artifacts remain redacted debugging -material. See [operations](../operations.md#debug-evidence) for placement +`llm-execution.json`. Request and response artifacts are debugging material. See [operations](../operations.md#debug-evidence) for placement and paths, [configuration](../config.md) for all settings, and the [CLI reference](../cli.md) for current flags. diff --git a/docs/operations.md b/docs/operations.md index ed4dd25..f639e45 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -48,11 +48,9 @@ existence as proof of success. The shared `--debug` flag enables retained artifacts in a run directory beneath `diagnostics.work_dir`; selecting a work directory alone does not enable capture. -Optional debug preparation or writes can produce redacted warnings without +Optional debug preparation or writes can produce warnings without changing correction decisions or the required output result. Debug material can -include transcript content, prompts, responses, and correction decisions. It is -not anonymized; protect it according to the input's sensitivity. Configured -credential values are redacted at the diagnostic boundary. Repair interactions +include transcript content, prompts, responses, and correction decisions. Protect this material according to the sensitivity of the inputs and responses. Repair interactions and revalidation artifacts include proposal and attempt coordinates; the correction ledger connects them to the original edit. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 6ed2c9e..8dadd92 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -111,6 +111,12 @@ Audit #3, #9, #31 (source bytes), #63–65. Use the bytes, malformed response evidence, actual paths, and native auth using synthetic data. Check feedback by decoding a production repair request. +**Complete.** Reports and debug artifacts encode typed data directly, source +snapshots preserve the acquired bytes, and repair requests carry the original +domain findings. Removed application credential discovery and substitution. +The CLI build and full offline suite passed with Go 1.27.3, including checks +for native authentication, malformed response capture, and destination paths. + ## Stage 3 — Unify optional interaction capture Audit #33 and #43. diff --git a/internal/cli/process.go b/internal/cli/process.go index 21b61ad..01eda77 100644 --- a/internal/cli/process.go +++ b/internal/cli/process.go @@ -36,7 +36,7 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom } prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output()) if err != nil { - return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err) + return commandkit.Failure(err.Error()).WithCause(err) } defer prepared.Prepared.Output.Close() var warningsMu sync.Mutex @@ -63,24 +63,24 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom warn(debug.WriteNormalizationSummary(summary)) } if err := inv.Context.Err(); err != nil { - return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err) + return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, err) } result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn) if runErr != nil { - return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr) + return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr) } encoder, err := outputschema.Resolve(cfg.Output.Schema) if err != nil { - return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err) + return finishProcess(inv, input, prepared, result.Snapshot, nil, err) } if result.Transcript == nil { - return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript")) + return finishProcess(inv, input, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript")) } payload, err := encoder.Encoder(result.Transcript) if err != nil { - return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err) + return finishProcess(inv, input, prepared, result.Snapshot, nil, err) } - return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil) + return finishProcess(inv, input, prepared, result.Snapshot, payload, nil) } // DebugPath returns the retained artifact directory, if its preparation succeeded. diff --git a/internal/cli/process_inputs.go b/internal/cli/process_inputs.go index 65cdbf3..c21a36a 100644 --- a/internal/cli/process_inputs.go +++ b/internal/cli/process_inputs.go @@ -87,10 +87,6 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu return nil, err } started := time.Now().UTC() - var secrets []string - if cfg.Reporting.Enabled || input.Debug.Value { - secrets = loaded.Runtime.ConfiguredSecrets(ctx) - } var initial diagnostics.ReportBundle var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle if cfg.Reporting.Enabled { @@ -98,7 +94,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu for index, item := range cfg.Pipeline.Stages { snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID)) } - initial, err = renderReportBundle(snapshot, secrets...) + initial, err = renderReportBundle(snapshot) if err != nil { return nil, err } @@ -109,7 +105,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu if primary != nil { failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}} } - bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...) + bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures)) return bundle } } @@ -120,7 +116,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu if err := loaded.Configuration.WriteEffective(&effective); err != nil { warnings = append(warnings, err) } - debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}} + debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}} } prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error { profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id) @@ -147,14 +143,14 @@ func generateRunID() (string, error) { return "run-" + hex.EncodeToString(random[:]), nil } -func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) { - errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...) +func renderReportBundle(value reporting.Snapshot) (diagnostics.ReportBundle, error) { + errorsYAML, err := reporting.YAML(value.ErrorsDocument()) if err != nil { return diagnostics.ReportBundle{}, err } - recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...) + recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument()) if err != nil { return diagnostics.ReportBundle{}, err } - return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil + return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value)}, nil } diff --git a/internal/cli/process_ordered_integration_test.go b/internal/cli/process_ordered_integration_test.go index b67530e..6c8cde0 100644 --- a/internal/cli/process_ordered_integration_test.go +++ b/internal/cli/process_ordered_integration_test.go @@ -147,7 +147,11 @@ pipeline: if readErr != nil { t.Fatal(readErr) } - if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) { + var compact bytes.Buffer + if err := json.Compact(&compact, raw); err != nil { + t.Fatalf("decode %s: %v", artifact, err) + } + if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) { t.Fatalf("%s missing attribution: %s", artifact, raw) } } diff --git a/internal/cli/process_output.go b/internal/cli/process_output.go index d932c34..539645f 100644 --- a/internal/cli/process_output.go +++ b/internal/cli/process_output.go @@ -7,19 +7,16 @@ import ( "time" "gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" commandkit "go.maximumdirect.net/commandkit" ) // finishProcess delivers the transcript and reports before emitting success information. -func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome { +func finishProcess(inv commandkit.Invocation, input processInput, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome { reporter := inv.Reporter() - secrets := loaded.Runtime.ConfiguredSecrets(inv.Context) - safe := redaction.New(secrets) for _, warning := range prepared.Prepared.Warnings { - _ = reporter.Warn("Debug artifact: " + safe.String(warning.Error())) + _ = reporter.Warn("Debug artifact: " + warning.Error()) } completed := time.Now().UTC() snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed @@ -36,7 +33,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle if len(reportPaths) > 0 { var err error - reports, err = renderReportBundle(snapshot, secrets...) + reports, err = renderReportBundle(snapshot) if err != nil { if primary == nil { primary = err @@ -44,7 +41,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces snapshot.Status = reporting.StatusFailed } failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle { - result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...) + result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures)) if bundleErr != nil { return reports } @@ -60,14 +57,14 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces } result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed) if result != nil { - return commandkit.Failure(safe.String(result.Error())).WithCause(result) + return commandkit.Failure(result.Error()).WithCause(result) } _ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot)) for _, path := range reportPaths { - _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path))) + _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", path)) } if prepared.Prepared.Debug != nil { - _ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path()))) + _ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", prepared.Prepared.Debug.Path())) } return commandkit.Success() } diff --git a/internal/cli/process_output_test.go b/internal/cli/process_output_test.go index 438d56e..812f30c 100644 --- a/internal/cli/process_output_test.go +++ b/internal/cli/process_output_test.go @@ -46,7 +46,7 @@ func TestPreparedDeliveryFollowsReportingMode(t *testing.T) { if err != nil { return commandkit.Failure(err.Error()) } - return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) + return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) }}) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) if err != nil { @@ -97,7 +97,7 @@ func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) { if err != nil { return commandkit.Failure(err.Error()) } - return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) + return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) }}) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) if err != nil { @@ -161,7 +161,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T) t.Fatal(err) } cancel() - return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil) + return finishProcess(inv, input, prepared, result.Snapshot, []byte("new transcript"), nil) }}) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) if err != nil { diff --git a/internal/cli/runtime_errors_test.go b/internal/cli/runtime_errors_test.go index 86a26e2..c52a336 100644 --- a/internal/cli/runtime_errors_test.go +++ b/internal/cli/runtime_errors_test.go @@ -6,7 +6,6 @@ import ( "errors" "os" "path/filepath" - "strings" "testing" commandkit "go.maximumdirect.net/commandkit" @@ -25,7 +24,6 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string { } func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { - t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret") definitions := map[string]string{ "malformed operator YAML": "profiles: [\n", "invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n", @@ -38,9 +36,6 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 { t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String()) } - if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") { - t.Fatal("configured secret leaked") - } }) } } diff --git a/internal/core/diagnostics/output_session.go b/internal/core/diagnostics/output_session.go index 2ffb269..cf61dba 100644 --- a/internal/core/diagnostics/output_session.go +++ b/internal/core/diagnostics/output_session.go @@ -43,7 +43,7 @@ func (f FileFailure) Error() string { return fmt.Sprintf("%s %q: %v", f.Kind, f. func (f FileFailure) Unwrap() error { return f.Err } // OutputSession owns checked destinations and per-file publication. Filesystem -// failures remain typed so CLI can present safe, redacted diagnostics. +// failures remain typed so CLI can present contextual diagnostics. type OutputSession struct { transcript string reports [3]string @@ -250,8 +250,8 @@ func (s *OutputSession) Initialize(initial ReportBundle, failed func([]FileFailu } // PrepareDebug creates optional initial diagnostics and returns any nonfatal problems. -func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte, secrets ...string) (*RunDirectory, []error) { - run, err := NewRunDirectory(workDir, runID, secrets...) +func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte) (*RunDirectory, []error) { + run, err := NewRunDirectory(workDir, runID) if err != nil { return nil, []error{err} } diff --git a/internal/core/diagnostics/output_session_test.go b/internal/core/diagnostics/output_session_test.go index a3d69e4..81091ff 100644 --- a/internal/core/diagnostics/output_session_test.go +++ b/internal/core/diagnostics/output_session_test.go @@ -2,13 +2,17 @@ package diagnostics import ( "bytes" + "encoding/json" "errors" "io" + "math" "os" "os/exec" "path/filepath" "strings" "testing" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) { @@ -242,7 +246,7 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) { if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) { t.Fatalf("unexpected parent: %v", err) } - run, err := NewRunDirectory(parent, "run-123", "secret") + run, err := NewRunDirectory(parent, "run-123") if err != nil { t.Fatal(err) } @@ -252,11 +256,11 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) { if _, err := NewRunDirectory(parent, "run-123"); err == nil { t.Fatal("duplicate run ID accepted") } - if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "secret"}); err != nil { + if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "edit"}); err != nil { t.Fatal(err) } contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger)) - if err != nil || strings.Contains(string(contents), "secret") { + if err != nil || string(contents) != "{\"proposal\":\"edit\"}\n" { t.Fatalf("debug artifact: %s %v", contents, err) } if _, err := os.Stat(run.Path()); err != nil { @@ -264,6 +268,48 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) { } } +func TestDebugTranscriptSnapshotsPreserveSourceAndTypedViews(t *testing.T) { + run, err := NewRunDirectory(t.TempDir(), "run-transcripts") + if err != nil { + t.Fatal(err) + } + raw := []byte(" [\n {\"id\": 2, \"text\": \"literal\", \"speaker\": \"A\", \"start\": 0, \"end\": 1}\n ] ") + source, err := schema.ParseSourceTranscriptJSON(raw) + if err != nil { + t.Fatal(err) + } + if err := run.WriteSourceTranscript(source, raw); err != nil { + t.Fatal(err) + } + normalized := &schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "normalized"}}} + if err := run.WriteNormalizedTranscript(normalized); err != nil { + t.Fatal(err) + } + acquired, err := os.ReadFile(filepath.Join(run.Path(), ArtifactSourceTranscript)) + if err != nil || !bytes.Equal(acquired, raw) { + t.Fatalf("acquired source = %q, error = %v", acquired, err) + } + for name, want := range map[string]string{ArtifactParsedSourceTranscript: "literal", ArtifactNormalizedTranscript: "normalized"} { + payload, err := os.ReadFile(filepath.Join(run.Path(), name)) + if err != nil { + t.Fatal(err) + } + var transcript schema.Transcript + if err := json.Unmarshal(payload, &transcript); err != nil { + t.Fatal(err) + } + if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 2 || transcript.Segments[0].Text != want { + t.Fatalf("%s: %s", name, payload) + } + } + if err := run.WriteJSONArtifact("invalid.json", math.NaN()); err == nil { + t.Fatal("expected serialization failure") + } + if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("failed artifact persisted: %v", err) + } +} + func TestSelectionAndAbortPreserveTranscript(t *testing.T) { dir := t.TempDir() output := filepath.Join(dir, "output.json") @@ -363,7 +409,7 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) { if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 { t.Fatalf("run=%v problems=%v", run, problems) } - run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("secret: sensitive\n"), "sensitive") + run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("version: 3\n")) if len(problems) != 0 { t.Fatal(problems) } @@ -371,8 +417,8 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(contents), "sensitive") { - t.Fatal("debug artifact contains secret") + if string(contents) != "version: 3\n" { + t.Fatalf("changed effective configuration: %q", contents) } } diff --git a/internal/core/diagnostics/run_dir.go b/internal/core/diagnostics/run_dir.go index 60f89ed..f327b82 100644 --- a/internal/core/diagnostics/run_dir.go +++ b/internal/core/diagnostics/run_dir.go @@ -9,13 +9,11 @@ import ( "time" "gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) // RunDirectory represents a per-run diagnostics directory type RunDirectory struct { - redactor redaction.Redactor path string createdAt time.Time } @@ -32,7 +30,7 @@ type InvocationMetadata struct { } // NewRunDirectory creates a new run directory under the configured work dir -func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, error) { +func NewRunDirectory(workDir, runID string) (*RunDirectory, error) { if strings.TrimSpace(workDir) == "" { return nil, fmt.Errorf("debug work directory is required") } @@ -53,7 +51,6 @@ func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, e return &RunDirectory{ path: runPath, - redactor: redaction.New(secrets), createdAt: time.Now().UTC(), }, nil } @@ -77,7 +74,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro } path := filepath.Join(r.path, ArtifactInvocationMetadata) - bytes, err := r.redactor.JSON(metadata) + bytes, err := json.Marshal(metadata) if err != nil { return fmt.Errorf("failed to marshal invocation metadata: %w", err) } @@ -95,25 +92,14 @@ func (r *RunDirectory) WriteEffectiveConfig(effectiveYAML []byte) error { // WriteSourceTranscript writes the source transcript artifact func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error { - // Preserve the source snapshot without copying unredacted input bytes. - var source any - decoder := json.NewDecoder(strings.NewReader(string(raw))) - decoder.UseNumber() - if err := decoder.Decode(&source); err != nil { - return fmt.Errorf("decode source snapshot: %w", err) - } - sourceBytes, err := r.redactor.JSON(source) - if err != nil { - return fmt.Errorf("redact source snapshot: %w", err) - } sourcePath := filepath.Join(r.path, ArtifactSourceTranscript) - if err := os.WriteFile(sourcePath, append(sourceBytes, '\n'), 0o644); err != nil { + if err := os.WriteFile(sourcePath, raw, 0o644); err != nil { return fmt.Errorf("failed to write source transcript: %w", err) } // Write parsed source for debugging parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript) - parsedBytes, err := r.redactor.JSON(transcript) + parsedBytes, err := json.Marshal(transcript) if err != nil { return fmt.Errorf("failed to marshal parsed source transcript: %w", err) } @@ -128,7 +114,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript // WriteNormalizedTranscript writes the normalized transcript artifact func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error { normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript) - bytes, err := r.redactor.JSON(transcript) + bytes, err := json.Marshal(transcript) if err != nil { return fmt.Errorf("failed to serialize normalized transcript: %w", err) } @@ -141,7 +127,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) // WriteNormalizationSummary writes the normalization summary artifact func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error { summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary) - bytes, err := r.redactor.JSON(summary) + bytes, err := json.Marshal(summary) if err != nil { return fmt.Errorf("failed to marshal normalization summary: %w", err) } @@ -154,7 +140,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error { artifactPath := filepath.Join(r.path, name) - bytes, err := r.redactor.JSON(payload) + bytes, err := json.Marshal(payload) if err != nil { return fmt.Errorf("failed to marshal %s: %w", name, err) } @@ -165,13 +151,8 @@ func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error { return nil } -// WriteYAMLArtifact writes a sanitized YAML diagnostic artifact. +// WriteYAMLArtifact writes the supplied YAML diagnostic artifact. func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error { - var err error - payload, err = r.redactor.YAML(payload) - if err != nil { - return err - } if err := os.WriteFile(filepath.Join(r.path, name), payload, 0o644); err != nil { return fmt.Errorf("failed to write %s: %w", name, err) } diff --git a/internal/core/redaction/redaction.go b/internal/core/redaction/redaction.go deleted file mode 100644 index 378ee1f..0000000 --- a/internal/core/redaction/redaction.go +++ /dev/null @@ -1,123 +0,0 @@ -// Package redaction sanitizes diagnostic representations without modifying domain data. -package redaction - -import ( - "bytes" - "encoding/json" - "fmt" - "sort" - "strings" - - "gopkg.in/yaml.v3" -) - -// Redactor replaces configured values in a single pass, longest matches first. -// Its zero value is usable and leaves strings unchanged. -type Redactor struct{ replacer *strings.Replacer } - -func New(secrets []string) Redactor { - values := map[string]bool{} - for _, secret := range secrets { - if secret == "" { - continue - } - values[secret] = true - // Provider content may itself be serialized JSON inside a string field. - encoded, _ := json.Marshal(secret) - values[string(encoded[1:len(encoded)-1])] = true - } - keys := make([]string, 0, len(values)) - for value := range values { - keys = append(keys, value) - } - sort.Slice(keys, func(i, j int) bool { - if len(keys[i]) == len(keys[j]) { - return keys[i] < keys[j] - } - return len(keys[i]) > len(keys[j]) - }) - pairs := make([]string, 0, 2*len(keys)) - for _, key := range keys { - pairs = append(pairs, key, "[REDACTED]") - } - return Redactor{replacer: strings.NewReplacer(pairs...)} -} - -func (r Redactor) String(value string) string { - if r.replacer == nil { - return value - } - return r.replacer.Replace(value) -} - -// JSON serializes first, then redacts only decoded string values and keys. -// Numbers retain their exact representation. Failure never returns original data. -func (r Redactor) JSON(value any) ([]byte, error) { - raw, err := json.Marshal(value) - if err != nil { - return nil, fmt.Errorf("serialize diagnostic: %s", r.String(err.Error())) - } - decoder := json.NewDecoder(bytes.NewReader(raw)) - decoder.UseNumber() - var decoded any - if err := decoder.Decode(&decoded); err != nil { - return nil, fmt.Errorf("decode diagnostic: %s", r.String(err.Error())) - } - safe, err := r.walk(decoded) - if err != nil { - return nil, err - } - return json.MarshalIndent(safe, "", " ") -} - -func (r Redactor) walk(value any) (any, error) { - switch v := value.(type) { - case string: - return r.String(v), nil - case []any: - for i := range v { - var err error - v[i], err = r.walk(v[i]) - if err != nil { - return nil, err - } - } - case map[string]any: - safe := make(map[string]any, len(v)) - for key, item := range v { - safeKey := r.String(key) - if _, exists := safe[safeKey]; exists { - return nil, fmt.Errorf("redaction would produce duplicate object keys") - } - safeItem, err := r.walk(item) - if err != nil { - return nil, err - } - safe[safeKey] = safeItem - } - return safe, nil - } - return value, nil -} - -// YAML preserves scalar types while sanitizing string nodes, including map keys. -func (r Redactor) YAML(raw []byte) ([]byte, error) { - var node yaml.Node - if err := yaml.Unmarshal(raw, &node); err != nil { - return nil, fmt.Errorf("decode diagnostic YAML: %s", r.String(err.Error())) - } - var visit func(*yaml.Node) - visit = func(n *yaml.Node) { - if n.Kind == yaml.ScalarNode && n.Tag == "!!str" { - n.Value = r.String(n.Value) - } - n.HeadComment = r.String(n.HeadComment) - n.LineComment = r.String(n.LineComment) - n.FootComment = r.String(n.FootComment) - for _, child := range n.Content { - visit(child) - } - } - visit(&node) - return yaml.Marshal(&node) -} diff --git a/internal/core/redaction/redaction_test.go b/internal/core/redaction/redaction_test.go deleted file mode 100644 index 8816d6c..0000000 --- a/internal/core/redaction/redaction_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package redaction_test - -import ( - "encoding/json" - "math" - "os" - "path/filepath" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" -) - -func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) { - secrets := []string{"2", `quoted"\key`, "overlap", "overlap-long", "REDACTED"} - r := redaction.New(secrets) - original := map[string]any{"number": 2, "large": json.Number("9007199254740993"), "bool": true, "text": `echo:2 quoted"\key overlap-long overlap`} - raw, err := r.JSON(original) - if err != nil { - t.Fatal(err) - } - var decoded map[string]any - if err := json.Unmarshal(raw, &decoded); err != nil { - t.Fatal(err) - } - if decoded["number"] != float64(2) || decoded["bool"] != true || !strings.Contains(string(raw), "9007199254740993") { - t.Fatalf("changed scalar meanings: %s", raw) - } - if decoded["text"] != "echo:[REDACTED] [REDACTED] [REDACTED] [REDACTED]" { - t.Fatalf("redaction: %s", raw) - } - if original["text"] != `echo:2 quoted"\key overlap-long overlap` { - t.Fatal("mutated domain data") - } - if raw, err := r.JSON(math.NaN()); err == nil || raw != nil { - t.Fatalf("unsafe fallback: %s %v", raw, err) - } - if raw, err := r.JSON(map[string]string{"overlap": "first", "overlap-long": "second"}); err == nil || raw != nil { - t.Fatalf("colliding redacted keys: %s %v", raw, err) - } - nested, _ := json.Marshal(map[string]string{"reason": `quoted"\key`}) - raw, err = r.JSON(map[string]string{"final_content": string(nested)}) - if err != nil || strings.Contains(string(raw), "key") { - t.Fatalf("nested content: %s %v", raw, err) - } - yaml, err := r.YAML([]byte("version: 2\nenabled: true\nmessage: echo:2\n")) - if err != nil || !strings.Contains(string(yaml), "version: 2") || !strings.Contains(string(yaml), "echo:[REDACTED]") { - t.Fatalf("YAML: %s %v", yaml, err) - } -} - -func TestDebugWritersRedactSourceAndNormalizedSnapshots(t *testing.T) { - dir := t.TempDir() - run, err := diagnostics.NewRunDirectory(dir, "run-test", "2") - if err != nil { - t.Fatal(err) - } - if err := run.WriteSourceTranscript(&schema.SourceTranscript{}, []byte(`{"segments":[{"text":"echo:2","id":2}]}`)); err != nil { - t.Fatal(err) - } - if err := run.WriteNormalizedTranscript(&schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "echo:2"}}}); err != nil { - t.Fatal(err) - } - for _, path := range []string{filepath.Join(run.Path(), diagnostics.ArtifactSourceTranscript), filepath.Join(run.Path(), diagnostics.ArtifactNormalizedTranscript)} { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(raw), "echo:2") || !strings.Contains(string(raw), "echo:[REDACTED]") || !strings.Contains(string(raw), `"id": 2`) { - t.Fatalf("snapshot: %s", raw) - } - } - if err := run.WriteJSONArtifact("invalid.json", map[string]any{"secret": "echo:2", "invalid": math.NaN()}); err == nil { - t.Fatal("expected serialization failure") - } - if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !os.IsNotExist(err) { - t.Fatalf("failed artifact persisted: %v", err) - } -} diff --git a/internal/core/reporting/markdown.go b/internal/core/reporting/markdown.go index 7f24818..666f180 100644 --- a/internal/core/reporting/markdown.go +++ b/internal/core/reporting/markdown.go @@ -5,8 +5,6 @@ import ( "html" "sort" "strings" - - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" ) // CompletionLine is the brief successful CLI summary. It uses the same counts @@ -24,12 +22,10 @@ func count(value *int) int { } // Markdown renders a fixed-layout human report from one processing snapshot. -func Markdown(snapshot Snapshot, secrets ...string) []byte { - r := redaction.New(secrets) - safe := func(value string) string { return escapeMarkdown(r.String(value)) } +func Markdown(snapshot Snapshot) []byte { var b strings.Builder - fmt.Fprintf(&b, "# Audita report: %s\n\n", safe(snapshot.RunID)) - fmt.Fprintf(&b, "Status: %s\n\n", safe(snapshot.Status)) + fmt.Fprintf(&b, "# Audita report: %s\n\n", escapeMarkdown(snapshot.RunID)) + fmt.Fprintf(&b, "Status: %s\n\n", escapeMarkdown(snapshot.Status)) fmt.Fprintf(&b, "Corrections applied: %d \n", count(snapshot.Summary.CorrectionsApplied)) fmt.Fprintf(&b, "Work units: %s total; %s completed; %s interrupted; %s not started \n", showCount(snapshot.Summary.WorkUnitsTotal), showCount(snapshot.Summary.WorkUnitsCompleted), @@ -38,20 +34,20 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte { fmt.Fprintf(&b, "Recoveries: %s of %s model operations \n", showCount(snapshot.Summary.Recoveries), showCount(snapshot.Summary.LLMOperations)) fmt.Fprintf(&b, "Proposal responses: %s; with filtering: %s\n\n", showCount(snapshot.Summary.ProposalResponses), showCount(snapshot.Summary.ProposalResponsesWithFiltering)) - fmt.Fprintf(&b, "Transcript input: %s \n", safe(snapshot.TranscriptPath)) + fmt.Fprintf(&b, "Transcript input: %s \n", escapeMarkdown(snapshot.TranscriptPath)) output := snapshot.OutputPath if output == "" { output = "stdout" } - fmt.Fprintf(&b, "Transcript output: %s\n\n", safe(output)) + fmt.Fprintf(&b, "Transcript output: %s\n\n", escapeMarkdown(output)) fmt.Fprintln(&b, "## Work-unit errors") fmt.Fprintln(&b) - writeMatrix(&b, snapshot, r, false) + writeMatrix(&b, snapshot, false) fmt.Fprintln(&b) fmt.Fprintln(&b, "## Recoveries") fmt.Fprintln(&b) - writeMatrix(&b, snapshot, r, true) + writeMatrix(&b, snapshot, true) fmt.Fprintln(&b) if len(snapshot.RepairOutcomes) > 0 { fmt.Fprintln(&b, "## Correction repair") @@ -59,14 +55,14 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte { fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |") fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |") for _, row := range snapshot.RepairOutcomes { - fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome)) + fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, escapeMarkdown(row.Outcome)) } fmt.Fprintln(&b) } fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.") fmt.Fprintln(&b) - writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe) - writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries), safe) + writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors)) + writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries)) fmt.Fprintln(&b, "A recovery can have multiple reasons; reason counts need not sum to the recovery total.") fmt.Fprintln(&b) fmt.Fprintln(&b, "Details: `errors.yml` and `recoveries.yml`.") @@ -80,7 +76,7 @@ func showCount(value *int) string { return fmt.Sprint(*value) } -func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redactor, recovery bool) { +func writeMatrix(b *strings.Builder, snapshot Snapshot, recovery bool) { if len(snapshot.StageNames) == 0 || len(snapshot.WorkUnits) == 0 { fmt.Fprintln(b, "No work units.") return @@ -88,7 +84,7 @@ func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redac columns := snapshot.StageNames fmt.Fprint(b, "| Chunk |") for _, name := range columns { - fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(redactor.String(name)))) + fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(name))) } fmt.Fprint(b, "\n| --- |") for range columns { @@ -137,7 +133,7 @@ var stageCellEscaper = strings.NewReplacer( "://", "://", "www.", "www.", "@", "@", ) -func writeReasons(b *strings.Builder, title string, counts map[string]int, safe func(string) string) { +func writeReasons(b *strings.Builder, title string, counts map[string]int) { fmt.Fprintf(b, "### %s\n\n", title) if len(counts) == 0 { fmt.Fprintln(b, "None.") @@ -150,7 +146,7 @@ func writeReasons(b *strings.Builder, title string, counts map[string]int, safe } sort.Strings(keys) for _, key := range keys { - fmt.Fprintf(b, "- `%s`: %d\n", safe(key), counts[key]) + fmt.Fprintf(b, "- `%s`: %d\n", escapeMarkdown(key), counts[key]) } fmt.Fprintln(b) } diff --git a/internal/core/reporting/markdown_test.go b/internal/core/reporting/markdown_test.go index 5947d87..0767b7e 100644 --- a/internal/core/reporting/markdown_test.go +++ b/internal/core/reporting/markdown_test.go @@ -28,7 +28,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) { Errors: []ErrorRecord{{Scope: "work_unit", Failures: []Failure{{Reason: "transport_failed"}, {Reason: "transport_failed"}}}}, Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}}, } - got := Markdown(snapshot, "secret") + got := Markdown(snapshot) text := string(got) // Compare only complete table rows, preserving order and states while leaving // prose and spacing outside the fixed matrices free to change. @@ -45,7 +45,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) { if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) { t.Fatalf("matrices: got %v; want %v", rows, wantRows) } - for _, fragment := range []string{"in\\|put\\`[REDACTED]\\`.json", "out
file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} { + for _, fragment := range []string{"in\\|put\\`secret\\`.json", "out
file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} { if !strings.Contains(text, fragment) { t.Fatalf("missing %q in:\n%s", fragment, text) } @@ -69,9 +69,6 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) { t.Fatalf("%s summary: got %v; want %v", label, got, want) } } - if strings.Contains(text, "secret") { - t.Fatal("secret leaked") - } var decoded struct { Summary Summary `yaml:"summary"` } @@ -88,12 +85,12 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) { } func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) { - text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret")) + text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}})) if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") { t.Fatal(text) } - text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret")) - if !strings.Contains(text, "bad\\|label\\`
\\[REDACTED\\]") { + text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}})) + if !strings.Contains(text, "bad\\|label\\`
secret") { t.Fatal(text) } } diff --git a/internal/core/reporting/report.go b/internal/core/reporting/report.go index 23145b7..c3ce3b2 100644 --- a/internal/core/reporting/report.go +++ b/internal/core/reporting/report.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" "gopkg.in/yaml.v3" ) @@ -178,17 +177,13 @@ func completionTime(t *time.Time) *string { return &value } -// YAML redacts only string nodes after typed serialization. -func YAML(document any, secrets ...string) ([]byte, error) { +// YAML encodes the report document. +func YAML(document any) ([]byte, error) { raw, err := yaml.Marshal(document) if err != nil { return nil, fmt.Errorf("encode report YAML: %w", err) } - safe, err := redaction.New(secrets).YAML(raw) - if err != nil { - return nil, fmt.Errorf("redact report YAML: %w", err) - } - return safe, nil + return raw, nil } // StageLabel identifies a configured entry by position, module, and optional ID. diff --git a/internal/core/reporting/report_test.go b/internal/core/reporting/report_test.go index a54e390..0eab73d 100644 --- a/internal/core/reporting/report_test.go +++ b/internal/core/reporting/report_test.go @@ -8,7 +8,7 @@ import ( "gopkg.in/yaml.v3" ) -func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) { +func TestYAMLPreservesScalarTypes(t *testing.T) { zero, one := 0, 1 falseValue := false snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC), @@ -16,18 +16,15 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) { Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}}, Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}} for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} { - payload, err := YAML(document, `secret\"key`, "1") + payload, err := YAML(document) if err != nil { t.Fatal(err) } - if strings.Contains(string(payload), `secret\"key`) { - t.Fatalf("secret leaked: %s", payload) - } var decoded map[string]any if err := yaml.Unmarshal(payload, &decoded); err != nil { t.Fatal(err) } - if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" { + if decoded["run_id"] != snapshot.RunID || decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" { t.Fatalf("wrong envelope types: %#v", decoded) } summary := decoded["summary"].(map[string]any) @@ -35,7 +32,7 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) { t.Fatalf("changed count types: %#v", summary) } } - if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") { + if payload, err := YAML(snapshot.RecoveriesDocument()); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") { t.Fatalf("boolean changed: %s %v", payload, err) } } diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index 709858f..c3ecb49 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -25,13 +25,11 @@ type CallHooks struct { } type RoleClient struct { - runtime *Runtime - role string - profileID string - settings config.LLM - hooks CallHooks - credentialMu sync.Mutex - credentialNames map[string]struct{} + runtime *Runtime + role string + profileID string + settings config.LLM + hooks CallHooks } var _ structuredoutput.Client = (*RoleClient)(nil) @@ -152,14 +150,6 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re var details pk.PreparedRun capturePrepared := func(_ context.Context, prepared pk.PreparedRun) { details = prepared - if name := details.EffectiveModelParams.APIKeyEnv; name != "" { - c.credentialMu.Lock() - if c.credentialNames == nil { - c.credentialNames = make(map[string]struct{}) - } - c.credentialNames[name] = struct{}{} - c.credentialMu.Unlock() - } response.Prompt = promptMetadata(details) record.Prompt = response.Prompt response.Model = details.EffectiveModelParams.Model @@ -311,17 +301,6 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)} } -// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials. -func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string { - c.credentialMu.Lock() - names := make([]string, 0, len(c.credentialNames)) - for name := range c.credentialNames { - names = append(names, name) - } - c.credentialMu.Unlock() - return c.runtime.ConfiguredSecrets(ctx, names...) -} - func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message { result := make([]structuredoutput.Message, len(messages)) for i, message := range messages { diff --git a/internal/framework/llm/diagnostics.go b/internal/framework/llm/diagnostics.go index 037b783..4e1c2a7 100644 --- a/internal/framework/llm/diagnostics.go +++ b/internal/framework/llm/diagnostics.go @@ -1,13 +1,12 @@ package llm import ( + "encoding/json" "fmt" "os" "path/filepath" "regexp" "strings" - - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" ) var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) @@ -23,13 +22,12 @@ type InteractionArtifacts struct { // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts. type DiagnosticsWriter struct { - dir string - redactor redaction.Redactor + dir string } // NewDiagnosticsWriter creates a diagnostics writer rooted at dir. -func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter { - return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)} +func NewDiagnosticsWriter(dir string) *DiagnosticsWriter { + return &DiagnosticsWriter{dir: dir} } // WriteInteraction writes request/response/error artifacts for a generic LLM @@ -82,12 +80,12 @@ func sanitizeStageName(stage string) string { } func (w *DiagnosticsWriter) writeJSON(path string, payload any) error { - redacted, err := w.redactor.JSON(payload) + encoded, err := json.Marshal(payload) if err != nil { return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err) } - redacted = append(redacted, '\n') - if err := os.WriteFile(path, redacted, 0o644); err != nil { + encoded = append(encoded, '\n') + if err := os.WriteFile(path, encoded, 0o644); err != nil { return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err) } return nil diff --git a/internal/framework/llm/diagnostics_test.go b/internal/framework/llm/diagnostics_test.go index 936a48f..aaa1041 100644 --- a/internal/framework/llm/diagnostics_test.go +++ b/internal/framework/llm/diagnostics_test.go @@ -4,13 +4,12 @@ import ( "encoding/json" "os" "path/filepath" - "strings" "testing" ) func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) { dir := t.TempDir() - writer := NewDiagnosticsWriter(dir, nil) + writer := NewDiagnosticsWriter(dir) artifacts, err := writer.WriteInteraction( "validation:batch-1", @@ -31,45 +30,6 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) { assertValidJSONFile(t, artifacts.ResponsePayloadPath) } -func TestDiagnosticsWriterRedactsSecrets(t *testing.T) { - dir := t.TempDir() - secret := "top-secret-token" - writer := NewDiagnosticsWriter(dir, []string{secret}) - - artifacts, err := writer.WriteInteraction( - "module glossary / request", - map[string]any{"authorization": "Bearer " + secret}, - map[string]any{"api_key": secret, "nested": map[string]any{"value": secret}}, - map[string]any{"echo": secret}, - map[string]any{"error": "failed with " + secret}, - ) - if err != nil { - t.Fatalf("WriteInteraction: %v", err) - } - - for _, path := range []string{ - artifacts.RequestMetadataPath, - artifacts.RequestPayloadPath, - artifacts.ResponsePayloadPath, - artifacts.ErrorPayloadPath, - } { - raw, readErr := os.ReadFile(path) - if readErr != nil { - t.Fatalf("read %s: %v", path, readErr) - } - content := string(raw) - if strings.Contains(content, secret) { - t.Fatalf("secret leaked in %s: %s", filepath.Base(path), content) - } - if !strings.Contains(content, "[REDACTED]") { - t.Fatalf("redaction marker missing in %s: %s", filepath.Base(path), content) - } - if !json.Valid(raw) { - t.Fatalf("invalid json in %s", filepath.Base(path)) - } - } -} - func assertValidJSONFile(t *testing.T, path string) { t.Helper() raw, err := os.ReadFile(path) diff --git a/internal/framework/llm/native_adapter_test.go b/internal/framework/llm/native_adapter_test.go index c7205f7..0d5ca09 100644 --- a/internal/framework/llm/native_adapter_test.go +++ b/internal/framework/llm/native_adapter_test.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "path/filepath" - "slices" "strings" "testing" "time" @@ -26,7 +25,7 @@ type transportFunc func(*http.Request) (*http.Response, error) func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } -func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) { +func TestRoleClientUsesSelectedNativeCredential(t *testing.T) { t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ") dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil { @@ -41,7 +40,7 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) { t.Fatal(err) } calls := 0 - providerErr := errors.New("provider echoed synthetic-secret-value") + providerErr := errors.New("transport stopped") httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) { calls++ if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" { @@ -62,10 +61,6 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) { if err == nil || calls != 1 { t.Fatalf("error = %v, calls = %d", err, calls) } - masked := runtime.RedactError(context.Background(), providerErr) - if strings.Contains(masked, "synthetic-secret-value") { - t.Fatal("credential not redacted") - } } func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) { @@ -282,10 +277,4 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) { } }) } - secrets := runtime.ConfiguredSecrets(context.Background()) - for _, value := range []string{"backend-secret-value", "profile-secret-value"} { - if !slices.Contains(secrets, value) { - t.Fatalf("missing credential masking value") - } - } } diff --git a/internal/framework/llm/runtime.go b/internal/framework/llm/runtime.go index 6a95568..b63da72 100644 --- a/internal/framework/llm/runtime.go +++ b/internal/framework/llm/runtime.go @@ -8,7 +8,6 @@ import ( "net/http" "path/filepath" "strings" - "sync" "sync/atomic" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -20,12 +19,10 @@ import ( // Runtime owns the captured engine shared by all processing clients. type Runtime struct { - Config config.Config - Engine *pk.Engine - SourcePaths []string - nextCallID atomic.Uint64 - credentialSourcesOnce sync.Once - credentialEnvNames map[string]struct{} + Config config.Config + Engine *pk.Engine + SourcePaths []string + nextCallID atomic.Uint64 } type AssetError struct{ Err error } diff --git a/internal/framework/llm/secrets.go b/internal/framework/llm/secrets.go deleted file mode 100644 index 53a13f1..0000000 --- a/internal/framework/llm/secrets.go +++ /dev/null @@ -1,78 +0,0 @@ -package llm - -import ( - "context" - "os" - "strings" - - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" -) - -// ConfiguredSecrets obtains only values referenced by captured credential metadata. -// It never selects a credential or treats an absent value as an error. -func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string { - if r == nil || r.Engine == nil { - return nil - } - r.credentialSourcesOnce.Do(func() { - r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx)) - }) - secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames)) - read := func(name string) { - if value := os.Getenv(name); value != "" { - secrets = append(secrets, value) - // PromptKit trims the value before sending it to the provider. - if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value { - secrets = append(secrets, trimmed) - } - } - } - for name := range r.credentialEnvNames { - read(name) - } - if len(additionalEnvNames) > 0 { - seen := make(map[string]struct{}, len(additionalEnvNames)) - for _, name := range additionalEnvNames { - if _, known := r.credentialEnvNames[name]; known { - continue - } - if _, duplicate := seen[name]; name == "" || duplicate { - continue - } - seen[name] = struct{}{} - read(name) - } - } - return secrets -} - -func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} { - names := make(map[string]struct{}) - add := func(name *string) { - if name != nil && *name != "" { - names[*name] = struct{}{} - } - } - if ids, err := r.Engine.ListProfiles(ctx); err == nil { - for _, id := range ids { - if profile, err := r.Engine.InspectProfile(ctx, id); err == nil { - add(profile.APIKeyEnv) - } - } - } - if ids, err := r.Engine.ListBackends(ctx); err == nil { - for _, id := range ids { - if backend, err := r.Engine.InspectBackend(ctx, id); err == nil { - add(backend.APIKeyEnv) - } - } - } - return names -} - -func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string { - if err == nil { - return "" - } - return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error()) -} diff --git a/internal/framework/llm/secrets_test.go b/internal/framework/llm/secrets_test.go deleted file mode 100644 index b25d6df..0000000 --- a/internal/framework/llm/secrets_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package llm - -import ( - "context" - "os" - "path/filepath" - "slices" - "testing" - - "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" - ck "go.maximumdirect.net/configkit" -) - -func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) { - var empty *Runtime - if empty.ConfiguredSecrets(context.Background()) != nil { - t.Fatal("nil runtime has secrets") - } - t.Setenv("SELECTED_KEY", "selected-secret") - t.Setenv("OTHER_KEY", "unselected-secret") - t.Setenv("UNRELATED_KEY", "unrelated-secret") - dir := t.TempDir() - for name, data := range map[string]string{ - "config.yml": "version: 3\n", - "prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n", - } { - if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil { - t.Fatal(err) - } - } - loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "") - if err != nil { - t.Fatal(err) - } - runtime, err := NewRuntime(context.Background(), loaded, nil) - if err != nil { - t.Fatal(err) - } - seen := map[string]bool{} - for _, value := range runtime.ConfiguredSecrets(context.Background()) { - seen[value] = true - } - if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] { - t.Fatalf("unexpected resolved backend secrets: %v", seen) - } - t.Setenv("SELECTED_KEY", "rotated-secret") - values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY") - slices.Sort(values) - if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) { - t.Fatalf("values did not reflect the new credential and prepared source: %v", values) - } -} diff --git a/internal/framework/preflight/preflight.go b/internal/framework/preflight/preflight.go index 3b76eb7..4a7fd84 100644 --- a/internal/framework/preflight/preflight.go +++ b/internal/framework/preflight/preflight.go @@ -28,7 +28,6 @@ type DebugInput struct { RunID string Metadata diagnostics.InvocationMetadata EffectiveConfig []byte - Secrets []string } type Prepared struct { @@ -85,7 +84,7 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) { } result := &Prepared{Output: session} if input.Debug != nil { - result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig, input.Debug.Secrets...) + result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig) } if err := ctx.Err(); err != nil { return nil, cancelOutput(err) diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index 2670c38..6d97523 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -130,17 +130,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { if req.DiagnosticsWriter != nil { writer = req.DiagnosticsWriter } else if strings.TrimSpace(req.DiagnosticsDir) != "" { - var secrets []string - if source, ok := req.LLMClient.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - secrets = source.ConfiguredSecrets(ctx) - } writer = diagnosticsWriterAdapter{ - writer: llm.NewDiagnosticsWriter( - filepath.Join(req.DiagnosticsDir, req.ModuleInstance), - secrets, - ), + writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)), } } diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 295be9d..434dda5 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -19,7 +19,6 @@ import ( ) type fakeStructuredClient struct { - secrets []string responses []StructuredCorrectionSet err error completion contracts.StructuredCompletionResponse @@ -50,8 +49,6 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr return response, nil } -func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } - type captureDiagnosticsWriter struct { err error lastStage string @@ -311,21 +308,16 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) { } } -func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { - primarySecret := "proposal-primary-secret" - validationSecret := "proposal-validation-secret" +func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) { client := &fakeStructuredClient{ responses: []StructuredCorrectionSet{ - {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}}, + {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}}, }, } - t.Setenv("PRIMARY_SECRET", primarySecret) - t.Setenv("VALIDATION_SECRET", validationSecret) - client.secrets = []string{primarySecret, validationSecret} req := defaultRequest(t) req.LLMClient = client req.DiagnosticsDir = t.TempDir() - req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret} + req.Variables = map[string]string{"SectionJSON": "section text"} got, err := GenerateCandidates(context.Background(), req) if err != nil { @@ -340,11 +332,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { if readErr != nil { t.Fatalf("read artifact %q: %v", path, readErr) } - if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { - t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw)) - } - if !strings.Contains(string(raw), "[REDACTED]") { - t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw)) + if !json.Valid(raw) { + t.Fatalf("invalid JSON artifact %q: %s", path, raw) } } diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 16989da..ff1d44f 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -7,8 +7,6 @@ import ( "path/filepath" "slices" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } } current := original - safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding { - findings := repairFindings(original.ProposalIndex, rejected) - redactor := redaction.New(repairSecrets(ctx, input)) - for i := range findings { - findings[i].Explanation = redactor.String(findings[i].Explanation) - } - return findings - } - findings := safeFindings(rejected) + findings := repairFindings(original.ProposalIndex, rejected) coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator) if !ok { return finish(fmt.Errorf("correction repair coordinator is unavailable")) @@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, if err != nil { failure = map[string]string{"error": err.Error()} } - writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input)) + writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)) artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure) if writeErr == nil { record.Artifacts = artifacts @@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, return nil, nil } if actionable(original.ProposalIndex, validated.rejected) { - findings = safeFindings(validated.rejected) + findings = repairFindings(original.ProposalIndex, validated.rejected) return buildRequest() } branch.Outcome = "unavailable" @@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, } return finish(nil) } - -func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string { - var secrets []string - for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} { - if source, ok := client.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - secrets = append(secrets, source.ConfiguredSecrets(ctx)...) - } - } - return secrets -} diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index acc1dd0..adda090 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -31,7 +31,6 @@ type repairClient struct { outputs []string calls []structuredoutput.Request err error - secrets []string } func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) { @@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil } -func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets } - func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) { client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}} candidates := []proposals.EnrichedCorrectionProposal{ @@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) { } } -func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) { - client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}} +func TestRepairArtifactsAreUnique(t *testing.T) { + client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}} candidates := []proposals.EnrichedCorrectionProposal{ {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, @@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) { if a == "" || a == b { t.Fatalf("artifact paths: %q %q", a, b) } - for _, path := range []string{a, b} { + for i, path := range []string{a, b} { data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } - if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") { - t.Fatalf("unredacted %s: %s", filepath.Base(path), data) + if !json.Valid(data) { + t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data) + } + var request proposal_generation.RepairRequest + if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil { + t.Fatal(err) + } + if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText || + len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." { + t.Fatalf("repair request for proposal %d: %+v", i, request) } } } @@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) { } } -func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) { - client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}} - validation := &repairClient{t: t, secrets: []string{"validation-secret"}} - candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()} - initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}} - _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") - if err != nil { - t.Fatal(err) - } - if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") { - t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"]) - } - data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(data), "validation-secret") { - t.Fatalf("credential in artifact: %s", data) - } -} - // rejectUntilRepair supplies a semantic rejection at the external validator seam. func rejectUntilRepair(preserved ...int) contracts.Validator { return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { @@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha dir := t.TempDir() for name, content := range map[string]string{ "config.yml": "version: 3\n", - "prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n", + "prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n", } { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { t.Fatal(err) diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index d370895..5f47fd5 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) { if interactionWriter == nil { interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)} - if source, ok := input.ValidationLLMClient.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) } - } } diagnosticsWriter = interactionWriter } @@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript { } type llmDiagnosticsWriterAdapter struct { - dir string - secrets func() []string + dir string } func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { - var secrets []string - if a.secrets != nil { - secrets = a.secrets() - } - art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) + art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) if err != nil { return validators.InteractionArtifacts{}, err } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 78e2b82..7a4c1c8 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "encoding/json" "errors" "os" "path/filepath" @@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { func ptrConfig(c config.Config) *config.Config { return &c } type fakeStructuredClient struct { - responses []validators.LLMValidationResponse - secrets []string - preparedSecret string - err error - calls int + responses []validators.LLMValidationResponse + err error + calls int } -func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } - func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { _ = ctx _ = req f.calls++ - if f.preparedSecret != "" { - f.secrets = append(f.secrets, f.preparedSecret) - f.preparedSecret = "" - } if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } @@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) { } } -func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { - primarySecret := "runner-primary-secret" - validationSecret := "runner-validation-secret" - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}} +func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) { + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) - client.secrets = []string{primarySecret} - client.preparedSecret = validationSecret cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ @@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { }}) out, err := r.Run(context.Background(), RunInput{ Config: &cfg, - Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}}, + Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, ValidationDiagnosticsDir: diagDir, }) @@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { if readErr != nil { t.Fatalf("read diagnostic %q: %v", path, readErr) } - if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { - t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw)) + if !json.Valid(raw) { + t.Fatalf("invalid diagnostic %q: %s", path, raw) } } @@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { if readErr != nil { t.Fatalf("read decision diagnostic: %v", readErr) } - if !strings.Contains(string(raw), "[REDACTED]") { - t.Fatalf("expected redaction marker in diagnostics") + if !strings.Contains(string(raw), "supported by context") { + t.Fatalf("missing decision evidence: %s", raw) } }