Remove application credential substitution

This commit is contained in:
2026-11-16 03:37:29 +00:00
parent e34672f7fe
commit ffe0e3491e
37 changed files with 217 additions and 729 deletions

View File

@@ -5,6 +5,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
@@ -118,14 +119,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
} }
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) { if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret) _, _ = io.WriteString(w, `{"error":{"message":"provider unavailable"}}`)
return return
} }
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}` content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
if p.mode == "invalid-credential-echo" { if p.mode == "malformed-response" {
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") content = `{"corrections": [`
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
content = string(raw)
} }
if p.mode == "repair-filter-later-failure" && !validation { if p.mode == "repair-filter-later-failure" && !validation {
if proposalCall == 1 { if proposalCall == 1 {
@@ -246,9 +245,6 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)} p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
if mode == "invalid-credential-echo" {
p.secret = " synthetic-echo-credential\t "
}
server := httptest.NewServer(http.HandlerFunc(p.serve)) server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close) t.Cleanup(server.Close)
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL) resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
@@ -957,40 +953,37 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 { if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
t.Fatalf("model failure report: %+v", document) t.Fatalf("model failure report: %+v", document)
} }
if strings.Contains(stderr, f.provider.secret) {
t.Fatal("secret leaked in stderr")
}
} }
func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) { func TestSubprocessCapturesMalformedProviderResponse(t *testing.T) {
f := newProcessFixture(t, "invalid-credential-echo", "[grammar]") f := newProcessFixture(t, "malformed-response", "[grammar]")
effective := strings.TrimSpace(f.provider.secret)
code, stdout, stderr := f.run(t, true, "--debug") code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") { if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr) t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
} }
if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) {
t.Fatal("credential escaped redaction in command output")
}
responseCaptured := false responseCaptured := false
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error { err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() { if err != nil {
return err return err
} }
if entry.IsDir() || !strings.HasSuffix(entry.Name(), "-response-payload.json") {
return nil
}
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
return err return err
} }
if bytes.Contains(data, []byte(effective)) { var response struct {
t.Errorf("credential escaped redaction in %s", entry.Name()) FinalContent string `json:"final_content"`
} }
if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) { if err := json.Unmarshal(data, &response); err != nil {
responseCaptured = true return err
} }
responseCaptured = responseCaptured || response.FinalContent == `{"corrections": [`
return nil return nil
}) })
if err != nil || !responseCaptured { if err != nil || !responseCaptured {
t.Fatalf("missing redacted provider echo: %v", err) t.Fatalf("missing malformed provider response: %v", err)
} }
} }
@@ -1246,27 +1239,16 @@ func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
} }
} }
func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) { func TestSubprocessAnnouncesActualOutputPaths(t *testing.T) {
for _, fail := range []bool{false, true} { for _, fail := range []bool{false, true} {
t.Run(fmt.Sprint(fail), func(t *testing.T) { t.Run(fmt.Sprint(fail), func(t *testing.T) {
f := newProcessFixture(t, "normal", "[]") f := newProcessFixture(t, "normal", "[]")
selected, unused := "selected-path-secret", "unused-path-secret" reportDir := filepath.Join(f.dir, "reports")
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
if err := os.Mkdir(reportDir, 0700); err != nil { if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err) t.Fatal(err)
} }
f.output = filepath.Join(reportDir, "out.json") f.output = filepath.Join(reportDir, "out.json")
debugParent := filepath.Join(f.dir, selected) debugParent := filepath.Join(f.dir, "diagnostics")
if fail { if fail {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil { if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -1276,11 +1258,8 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
if (!fail && code != 0) || (fail && code != 2) { if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr) t.Fatalf("code=%d stderr=%q", code, stderr)
} }
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) { if !fail && (!strings.Contains(stderr, filepath.Join(reportDir, "report.md")) || !strings.Contains(stderr, debugParent)) {
t.Fatalf("credential in stderr: %q", stderr) t.Fatalf("missing destination announcement: %q", stderr)
}
if !fail && !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
} }
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil { if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -11,11 +11,10 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
applicable. Preserve the precedence defined in the applicable. Preserve the precedence defined in the
[configuration contract](../config.md#loading-and-composition), including the [configuration contract](../config.md#loading-and-composition), including the
distinct validation and effective-config commands. distinct validation and effective-config commands.
3. Update redaction when secrets are involved. 3. Review configuration tests and related CLI tests for precedence, validation,
4. Review configuration tests and related CLI tests for precedence, validation,
and source-specific behavior. Add or adjust tests for the changed contract and source-specific behavior. Add or adjust tests for the changed contract
according to the [testing policy](../policy/testing.md). according to the [testing policy](../policy/testing.md).
5. Update the configuration reference and affected [examples](../../examples/). 4. Update the configuration reference and affected [examples](../../examples/).
## Changing CLI Behavior ## Changing CLI Behavior
@@ -51,8 +50,7 @@ Select checks according to the [testing policy](../policy/testing.md):
- Process/output changes: verify stdout, stderr, exit, output-file, and report - Process/output changes: verify stdout, stderr, exit, output-file, and report
behavior against the [CLI contract](../cli.md) and behavior against the [CLI contract](../cli.md) and
[subprocess guidance](../integrations/subprocess.md), including failures. [subprocess guidance](../integrations/subprocess.md), including failures.
- Report or diagnostics changes: review [report assembly tests and redaction - Report or diagnostics changes: review [report assembly tests](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests) - Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
and `internal/core/schema` tests, including supported formats and rejection of and `internal/core/schema` tests, including supported formats and rejection of
unsupported schemas. unsupported schemas.

View File

@@ -12,8 +12,7 @@ work-unit errors, separate run errors, and one recovery per successful logical
operation. Reports order stages, sections, validators, and batches by semantic operation. Reports order stages, sections, validators, and batches by semantic
identity, never concurrent completion time. The core reporting package owns identity, never concurrent completion time. The core reporting package owns
Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI
completion formatter. YAML redaction walks serialized string nodes so numeric completion formatter. Reports encode the typed documents directly to YAML. Ordinary reports omit raw causes,
and boolean values retain their types. Ordinary reports omit raw causes,
transcript text, complete decisions, and token/timing payloads. transcript text, complete decisions, and token/timing payloads.
Validation operations with unavailable decisions report only affected requested Validation operations with unavailable decisions report only affected requested
indices, even when other candidates in the batch continue. Ignored extra indices, even when other candidates in the batch continue. Ignored extra
@@ -34,17 +33,16 @@ pass; a published transcript is never erased by a later report failure.
Debug is independent. The shared `--debug` flag enables a unique retained child Debug is independent. The shared `--debug` flag enables a unique retained child
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
when debug is off. With debug enabled, it captures source and normalized when debug is off. With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized
snapshots, invocation, effective settings, correction ledger, LLM execution transcript snapshots. It also captures invocation metadata, effective settings
observations, and model interactions. Candidate capture includes proposals later rendered by ConfigKit, the correction ledger, LLM execution observations, and
model interactions. Candidate capture includes proposals later
rejected or interrupted. Repair and revalidation artifacts use section, proposal, rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable; their retained content attempt, validator, and batch coordinates as applicable. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work.
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
replacements and distinguishes unavailable candidates from interrupted work.
A concurrency-safe diagnostic callback reports optional write failures without A concurrency-safe diagnostic callback reports optional write failures without
aborting processing. Diagnostic redaction uses credential source metadata; debug aborting processing. Debug artifacts can contain sensitive transcript and
artifacts can still contain sensitive transcript and prompt content. Execution prompt content. Execution
position and optional operator ID are carried separately from generated safe position and optional operator ID are carried separately from generated safe
artifact path coordinates. artifact path coordinates.

View File

@@ -15,9 +15,7 @@ The operation budget includes preparation, admission, retries, and structural
repair. `ErrOperationTimeout` remains recoverable even when wrapped by a repair. `ErrOperationTimeout` remains recoverable even when wrapped by a
preparation error. Parent cancellation remains distinct. The prepared notification preparation error. Parent cancellation remains distinct. The prepared notification
provides the actual request snapshot; failures before notification have no snapshot. provides the actual request snapshot; failures before notification have no snapshot.
PromptKit owns backend and credential selection. The adapter uses captured PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md)
credential source names for artifact redaction without inspecting unrelated
environment values. See the [integration contract](../integrations/openai-compatible-llm.md)
for failure behavior and the [configuration reference](../config.md) for sources. for failure behavior and the [configuration reference](../config.md) for sources.
When correction repair is enabled, the proposal call captures a native immutable When correction repair is enabled, the proposal call captures a native immutable

View File

@@ -67,7 +67,7 @@ artifact lifecycle.
## Component Responsibilities ## Component Responsibilities
Core components implement configuration defaults/loading/validation/redaction, Core components implement configuration defaults, loading, and validation,
transcript schemas and derived glossary views, normalization and chunking, output encoding, transcript schemas and derived glossary views, normalization and chunking, output encoding,
diagnostics artifact helpers, and public report structures. diagnostics artifact helpers, and public report structures.

View File

@@ -121,8 +121,7 @@ provider attempts. With debug enabled, utilization remains grouped by backend
capacity and usage fields retain known/complete flags. Debug runs write capacity and usage fields retain known/complete flags. Debug runs write
`effective-config.redacted.yaml`, `selected-profiles.yaml`, `effective-config.redacted.yaml`, `selected-profiles.yaml`,
`config-origins.yaml`, `utilization-diagnostics.json`, and `config-origins.yaml`, `utilization-diagnostics.json`, and
`llm-execution.json`. Request and response artifacts remain redacted debugging `llm-execution.json`. Request and response artifacts are debugging material. See [operations](../operations.md#debug-evidence) for placement
material. See [operations](../operations.md#debug-evidence) for placement
and paths, [configuration](../config.md) for all settings, and the and paths, [configuration](../config.md) for all settings, and the
[CLI reference](../cli.md) for current flags. [CLI reference](../cli.md) for current flags.

View File

@@ -48,11 +48,9 @@ existence as proof of success.
The shared `--debug` flag enables retained artifacts in a run directory beneath The shared `--debug` flag enables retained artifacts in a run directory beneath
`diagnostics.work_dir`; selecting a work directory alone does not enable capture. `diagnostics.work_dir`; selecting a work directory alone does not enable capture.
Optional debug preparation or writes can produce redacted warnings without Optional debug preparation or writes can produce warnings without
changing correction decisions or the required output result. Debug material can changing correction decisions or the required output result. Debug material can
include transcript content, prompts, responses, and correction decisions. It is include transcript content, prompts, responses, and correction decisions. Protect this material according to the sensitivity of the inputs and responses. Repair interactions
not anonymized; protect it according to the input's sensitivity. Configured
credential values are redacted at the diagnostic boundary. Repair interactions
and revalidation artifacts include proposal and attempt coordinates; the and revalidation artifacts include proposal and attempt coordinates; the
correction ledger connects them to the original edit. correction ledger connects them to the original edit.

View File

@@ -111,6 +111,12 @@ Audit #3, #9, #31 (source bytes), #63–65. Use the
bytes, malformed response evidence, actual paths, and native auth using bytes, malformed response evidence, actual paths, and native auth using
synthetic data. Check feedback by decoding a production repair request. synthetic data. Check feedback by decoding a production repair request.
**Complete.** Reports and debug artifacts encode typed data directly, source
snapshots preserve the acquired bytes, and repair requests carry the original
domain findings. Removed application credential discovery and substitution.
The CLI build and full offline suite passed with Go 1.27.3, including checks
for native authentication, malformed response capture, and destination paths.
## Stage 3 — Unify optional interaction capture ## Stage 3 — Unify optional interaction capture
Audit #33 and #43. Audit #33 and #43.

View File

@@ -36,7 +36,7 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
} }
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output()) prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil { if err != nil {
return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err) return commandkit.Failure(err.Error()).WithCause(err)
} }
defer prepared.Prepared.Output.Close() defer prepared.Prepared.Output.Close()
var warningsMu sync.Mutex var warningsMu sync.Mutex
@@ -63,24 +63,24 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
warn(debug.WriteNormalizationSummary(summary)) warn(debug.WriteNormalizationSummary(summary))
} }
if err := inv.Context.Err(); err != nil { if err := inv.Context.Err(); err != nil {
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err) return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, err)
} }
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn) result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
if runErr != nil { if runErr != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr) return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
} }
encoder, err := outputschema.Resolve(cfg.Output.Schema) encoder, err := outputschema.Resolve(cfg.Output.Schema)
if err != nil { if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err) return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
} }
if result.Transcript == nil { if result.Transcript == nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript")) return finishProcess(inv, input, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
} }
payload, err := encoder.Encoder(result.Transcript) payload, err := encoder.Encoder(result.Transcript)
if err != nil { if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err) return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
} }
return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil) return finishProcess(inv, input, prepared, result.Snapshot, payload, nil)
} }
// DebugPath returns the retained artifact directory, if its preparation succeeded. // DebugPath returns the retained artifact directory, if its preparation succeeded.

View File

@@ -87,10 +87,6 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
return nil, err return nil, err
} }
started := time.Now().UTC() started := time.Now().UTC()
var secrets []string
if cfg.Reporting.Enabled || input.Debug.Value {
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
}
var initial diagnostics.ReportBundle var initial diagnostics.ReportBundle
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
if cfg.Reporting.Enabled { if cfg.Reporting.Enabled {
@@ -98,7 +94,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
for index, item := range cfg.Pipeline.Stages { for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID)) snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
} }
initial, err = renderReportBundle(snapshot, secrets...) initial, err = renderReportBundle(snapshot)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -109,7 +105,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
if primary != nil { if primary != nil {
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}} failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
} }
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...) bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures))
return bundle return bundle
} }
} }
@@ -120,7 +116,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
if err := loaded.Configuration.WriteEffective(&effective); err != nil { if err := loaded.Configuration.WriteEffective(&effective); err != nil {
warnings = append(warnings, err) warnings = append(warnings, err)
} }
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}} debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
} }
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error { prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id) profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
@@ -147,14 +143,14 @@ func generateRunID() (string, error) {
return "run-" + hex.EncodeToString(random[:]), nil return "run-" + hex.EncodeToString(random[:]), nil
} }
func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) { func renderReportBundle(value reporting.Snapshot) (diagnostics.ReportBundle, error) {
errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...) errorsYAML, err := reporting.YAML(value.ErrorsDocument())
if err != nil { if err != nil {
return diagnostics.ReportBundle{}, err return diagnostics.ReportBundle{}, err
} }
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...) recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument())
if err != nil { if err != nil {
return diagnostics.ReportBundle{}, err return diagnostics.ReportBundle{}, err
} }
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value)}, nil
} }

View File

@@ -147,7 +147,11 @@ pipeline:
if readErr != nil { if readErr != nil {
t.Fatal(readErr) t.Fatal(readErr)
} }
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) { var compact bytes.Buffer
if err := json.Compact(&compact, raw); err != nil {
t.Fatalf("decode %s: %v", artifact, err)
}
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw) t.Fatalf("%s missing attribution: %s", artifact, raw)
} }
} }

View File

@@ -7,19 +7,16 @@ import (
"time" "time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics" "gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
commandkit "go.maximumdirect.net/commandkit" commandkit "go.maximumdirect.net/commandkit"
) )
// finishProcess delivers the transcript and reports before emitting success information. // finishProcess delivers the transcript and reports before emitting success information.
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome { func finishProcess(inv commandkit.Invocation, input processInput, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
reporter := inv.Reporter() reporter := inv.Reporter()
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
safe := redaction.New(secrets)
for _, warning := range prepared.Prepared.Warnings { for _, warning := range prepared.Prepared.Warnings {
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error())) _ = reporter.Warn("Debug artifact: " + warning.Error())
} }
completed := time.Now().UTC() completed := time.Now().UTC()
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
@@ -36,7 +33,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
if len(reportPaths) > 0 { if len(reportPaths) > 0 {
var err error var err error
reports, err = renderReportBundle(snapshot, secrets...) reports, err = renderReportBundle(snapshot)
if err != nil { if err != nil {
if primary == nil { if primary == nil {
primary = err primary = err
@@ -44,7 +41,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
snapshot.Status = reporting.StatusFailed snapshot.Status = reporting.StatusFailed
} }
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle { failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...) result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures))
if bundleErr != nil { if bundleErr != nil {
return reports return reports
} }
@@ -60,14 +57,14 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
} }
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed) result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil { if result != nil {
return commandkit.Failure(safe.String(result.Error())).WithCause(result) return commandkit.Failure(result.Error()).WithCause(result)
} }
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot)) _ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range reportPaths { for _, path := range reportPaths {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path))) _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", path))
} }
if prepared.Prepared.Debug != nil { if prepared.Prepared.Debug != nil {
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path()))) _ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", prepared.Prepared.Debug.Path()))
} }
return commandkit.Success() return commandkit.Success()
} }

View File

@@ -46,7 +46,7 @@ func TestPreparedDeliveryFollowsReportingMode(t *testing.T) {
if err != nil { if err != nil {
return commandkit.Failure(err.Error()) return commandkit.Failure(err.Error())
} }
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}}) }})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil { if err != nil {
@@ -97,7 +97,7 @@ func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) {
if err != nil { if err != nil {
return commandkit.Failure(err.Error()) return commandkit.Failure(err.Error())
} }
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil) return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}}) }})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil { if err != nil {
@@ -161,7 +161,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
t.Fatal(err) t.Fatal(err)
} }
cancel() cancel()
return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil) return finishProcess(inv, input, prepared, result.Snapshot, []byte("new transcript"), nil)
}}) }})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd) app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil { if err != nil {

View File

@@ -6,7 +6,6 @@ import (
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
commandkit "go.maximumdirect.net/commandkit" commandkit "go.maximumdirect.net/commandkit"
@@ -25,7 +24,6 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
} }
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
definitions := map[string]string{ definitions := map[string]string{
"malformed operator YAML": "profiles: [\n", "malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n", "invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
@@ -38,9 +36,6 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 { if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String()) t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
} }
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatal("configured secret leaked")
}
}) })
} }
} }

View File

@@ -43,7 +43,7 @@ func (f FileFailure) Error() string { return fmt.Sprintf("%s %q: %v", f.Kind, f.
func (f FileFailure) Unwrap() error { return f.Err } func (f FileFailure) Unwrap() error { return f.Err }
// OutputSession owns checked destinations and per-file publication. Filesystem // OutputSession owns checked destinations and per-file publication. Filesystem
// failures remain typed so CLI can present safe, redacted diagnostics. // failures remain typed so CLI can present contextual diagnostics.
type OutputSession struct { type OutputSession struct {
transcript string transcript string
reports [3]string reports [3]string
@@ -250,8 +250,8 @@ func (s *OutputSession) Initialize(initial ReportBundle, failed func([]FileFailu
} }
// PrepareDebug creates optional initial diagnostics and returns any nonfatal problems. // PrepareDebug creates optional initial diagnostics and returns any nonfatal problems.
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte, secrets ...string) (*RunDirectory, []error) { func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte) (*RunDirectory, []error) {
run, err := NewRunDirectory(workDir, runID, secrets...) run, err := NewRunDirectory(workDir, runID)
if err != nil { if err != nil {
return nil, []error{err} return nil, []error{err}
} }

View File

@@ -2,13 +2,17 @@ package diagnostics
import ( import (
"bytes" "bytes"
"encoding/json"
"errors" "errors"
"io" "io"
"math"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
) )
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) { func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
@@ -242,7 +246,7 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("unexpected parent: %v", err) t.Fatalf("unexpected parent: %v", err)
} }
run, err := NewRunDirectory(parent, "run-123", "secret") run, err := NewRunDirectory(parent, "run-123")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -252,11 +256,11 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
if _, err := NewRunDirectory(parent, "run-123"); err == nil { if _, err := NewRunDirectory(parent, "run-123"); err == nil {
t.Fatal("duplicate run ID accepted") t.Fatal("duplicate run ID accepted")
} }
if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "secret"}); err != nil { if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "edit"}); err != nil {
t.Fatal(err) t.Fatal(err)
} }
contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger)) contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger))
if err != nil || strings.Contains(string(contents), "secret") { if err != nil || string(contents) != "{\"proposal\":\"edit\"}\n" {
t.Fatalf("debug artifact: %s %v", contents, err) t.Fatalf("debug artifact: %s %v", contents, err)
} }
if _, err := os.Stat(run.Path()); err != nil { if _, err := os.Stat(run.Path()); err != nil {
@@ -264,6 +268,48 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
} }
} }
func TestDebugTranscriptSnapshotsPreserveSourceAndTypedViews(t *testing.T) {
run, err := NewRunDirectory(t.TempDir(), "run-transcripts")
if err != nil {
t.Fatal(err)
}
raw := []byte(" [\n {\"id\": 2, \"text\": \"literal\", \"speaker\": \"A\", \"start\": 0, \"end\": 1}\n ] ")
source, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
if err := run.WriteSourceTranscript(source, raw); err != nil {
t.Fatal(err)
}
normalized := &schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "normalized"}}}
if err := run.WriteNormalizedTranscript(normalized); err != nil {
t.Fatal(err)
}
acquired, err := os.ReadFile(filepath.Join(run.Path(), ArtifactSourceTranscript))
if err != nil || !bytes.Equal(acquired, raw) {
t.Fatalf("acquired source = %q, error = %v", acquired, err)
}
for name, want := range map[string]string{ArtifactParsedSourceTranscript: "literal", ArtifactNormalizedTranscript: "normalized"} {
payload, err := os.ReadFile(filepath.Join(run.Path(), name))
if err != nil {
t.Fatal(err)
}
var transcript schema.Transcript
if err := json.Unmarshal(payload, &transcript); err != nil {
t.Fatal(err)
}
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 2 || transcript.Segments[0].Text != want {
t.Fatalf("%s: %s", name, payload)
}
}
if err := run.WriteJSONArtifact("invalid.json", math.NaN()); err == nil {
t.Fatal("expected serialization failure")
}
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("failed artifact persisted: %v", err)
}
}
func TestSelectionAndAbortPreserveTranscript(t *testing.T) { func TestSelectionAndAbortPreserveTranscript(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
output := filepath.Join(dir, "output.json") output := filepath.Join(dir, "output.json")
@@ -363,7 +409,7 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 { if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 {
t.Fatalf("run=%v problems=%v", run, problems) t.Fatalf("run=%v problems=%v", run, problems)
} }
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("secret: sensitive\n"), "sensitive") run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("version: 3\n"))
if len(problems) != 0 { if len(problems) != 0 {
t.Fatal(problems) t.Fatal(problems)
} }
@@ -371,8 +417,8 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if strings.Contains(string(contents), "sensitive") { if string(contents) != "version: 3\n" {
t.Fatal("debug artifact contains secret") t.Fatalf("changed effective configuration: %q", contents)
} }
} }

View File

@@ -9,13 +9,11 @@ import (
"time" "time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization" "gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
) )
// RunDirectory represents a per-run diagnostics directory // RunDirectory represents a per-run diagnostics directory
type RunDirectory struct { type RunDirectory struct {
redactor redaction.Redactor
path string path string
createdAt time.Time createdAt time.Time
} }
@@ -32,7 +30,7 @@ type InvocationMetadata struct {
} }
// NewRunDirectory creates a new run directory under the configured work dir // NewRunDirectory creates a new run directory under the configured work dir
func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, error) { func NewRunDirectory(workDir, runID string) (*RunDirectory, error) {
if strings.TrimSpace(workDir) == "" { if strings.TrimSpace(workDir) == "" {
return nil, fmt.Errorf("debug work directory is required") return nil, fmt.Errorf("debug work directory is required")
} }
@@ -53,7 +51,6 @@ func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, e
return &RunDirectory{ return &RunDirectory{
path: runPath, path: runPath,
redactor: redaction.New(secrets),
createdAt: time.Now().UTC(), createdAt: time.Now().UTC(),
}, nil }, nil
} }
@@ -77,7 +74,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
} }
path := filepath.Join(r.path, ArtifactInvocationMetadata) path := filepath.Join(r.path, ArtifactInvocationMetadata)
bytes, err := r.redactor.JSON(metadata) bytes, err := json.Marshal(metadata)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal invocation metadata: %w", err) return fmt.Errorf("failed to marshal invocation metadata: %w", err)
} }
@@ -95,25 +92,14 @@ func (r *RunDirectory) WriteEffectiveConfig(effectiveYAML []byte) error {
// WriteSourceTranscript writes the source transcript artifact // WriteSourceTranscript writes the source transcript artifact
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error { func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
// Preserve the source snapshot without copying unredacted input bytes.
var source any
decoder := json.NewDecoder(strings.NewReader(string(raw)))
decoder.UseNumber()
if err := decoder.Decode(&source); err != nil {
return fmt.Errorf("decode source snapshot: %w", err)
}
sourceBytes, err := r.redactor.JSON(source)
if err != nil {
return fmt.Errorf("redact source snapshot: %w", err)
}
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript) sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
if err := os.WriteFile(sourcePath, append(sourceBytes, '\n'), 0o644); err != nil { if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
return fmt.Errorf("failed to write source transcript: %w", err) return fmt.Errorf("failed to write source transcript: %w", err)
} }
// Write parsed source for debugging // Write parsed source for debugging
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript) parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
parsedBytes, err := r.redactor.JSON(transcript) parsedBytes, err := json.Marshal(transcript)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal parsed source transcript: %w", err) return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
} }
@@ -128,7 +114,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
// WriteNormalizedTranscript writes the normalized transcript artifact // WriteNormalizedTranscript writes the normalized transcript artifact
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error { func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript) normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
bytes, err := r.redactor.JSON(transcript) bytes, err := json.Marshal(transcript)
if err != nil { if err != nil {
return fmt.Errorf("failed to serialize normalized transcript: %w", err) return fmt.Errorf("failed to serialize normalized transcript: %w", err)
} }
@@ -141,7 +127,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
// WriteNormalizationSummary writes the normalization summary artifact // WriteNormalizationSummary writes the normalization summary artifact
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error { func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary) summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
bytes, err := r.redactor.JSON(summary) bytes, err := json.Marshal(summary)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal normalization summary: %w", err) return fmt.Errorf("failed to marshal normalization summary: %w", err)
} }
@@ -154,7 +140,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error { func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
artifactPath := filepath.Join(r.path, name) artifactPath := filepath.Join(r.path, name)
bytes, err := r.redactor.JSON(payload) bytes, err := json.Marshal(payload)
if err != nil { if err != nil {
return fmt.Errorf("failed to marshal %s: %w", name, err) return fmt.Errorf("failed to marshal %s: %w", name, err)
} }
@@ -165,13 +151,8 @@ func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
return nil return nil
} }
// WriteYAMLArtifact writes a sanitized YAML diagnostic artifact. // WriteYAMLArtifact writes the supplied YAML diagnostic artifact.
func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error { func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error {
var err error
payload, err = r.redactor.YAML(payload)
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(r.path, name), payload, 0o644); err != nil { if err := os.WriteFile(filepath.Join(r.path, name), payload, 0o644); err != nil {
return fmt.Errorf("failed to write %s: %w", name, err) return fmt.Errorf("failed to write %s: %w", name, err)
} }

View File

@@ -1,123 +0,0 @@
// Package redaction sanitizes diagnostic representations without modifying domain data.
package redaction
import (
"bytes"
"encoding/json"
"fmt"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
// Redactor replaces configured values in a single pass, longest matches first.
// Its zero value is usable and leaves strings unchanged.
type Redactor struct{ replacer *strings.Replacer }
func New(secrets []string) Redactor {
values := map[string]bool{}
for _, secret := range secrets {
if secret == "" {
continue
}
values[secret] = true
// Provider content may itself be serialized JSON inside a string field.
encoded, _ := json.Marshal(secret)
values[string(encoded[1:len(encoded)-1])] = true
}
keys := make([]string, 0, len(values))
for value := range values {
keys = append(keys, value)
}
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) == len(keys[j]) {
return keys[i] < keys[j]
}
return len(keys[i]) > len(keys[j])
})
pairs := make([]string, 0, 2*len(keys))
for _, key := range keys {
pairs = append(pairs, key, "[REDACTED]")
}
return Redactor{replacer: strings.NewReplacer(pairs...)}
}
func (r Redactor) String(value string) string {
if r.replacer == nil {
return value
}
return r.replacer.Replace(value)
}
// JSON serializes first, then redacts only decoded string values and keys.
// Numbers retain their exact representation. Failure never returns original data.
func (r Redactor) JSON(value any) ([]byte, error) {
raw, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize diagnostic: %s", r.String(err.Error()))
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
var decoded any
if err := decoder.Decode(&decoded); err != nil {
return nil, fmt.Errorf("decode diagnostic: %s", r.String(err.Error()))
}
safe, err := r.walk(decoded)
if err != nil {
return nil, err
}
return json.MarshalIndent(safe, "", " ")
}
func (r Redactor) walk(value any) (any, error) {
switch v := value.(type) {
case string:
return r.String(v), nil
case []any:
for i := range v {
var err error
v[i], err = r.walk(v[i])
if err != nil {
return nil, err
}
}
case map[string]any:
safe := make(map[string]any, len(v))
for key, item := range v {
safeKey := r.String(key)
if _, exists := safe[safeKey]; exists {
return nil, fmt.Errorf("redaction would produce duplicate object keys")
}
safeItem, err := r.walk(item)
if err != nil {
return nil, err
}
safe[safeKey] = safeItem
}
return safe, nil
}
return value, nil
}
// YAML preserves scalar types while sanitizing string nodes, including map keys.
func (r Redactor) YAML(raw []byte) ([]byte, error) {
var node yaml.Node
if err := yaml.Unmarshal(raw, &node); err != nil {
return nil, fmt.Errorf("decode diagnostic YAML: %s", r.String(err.Error()))
}
var visit func(*yaml.Node)
visit = func(n *yaml.Node) {
if n.Kind == yaml.ScalarNode && n.Tag == "!!str" {
n.Value = r.String(n.Value)
}
n.HeadComment = r.String(n.HeadComment)
n.LineComment = r.String(n.LineComment)
n.FootComment = r.String(n.FootComment)
for _, child := range n.Content {
visit(child)
}
}
visit(&node)
return yaml.Marshal(&node)
}

View File

@@ -1,81 +0,0 @@
package redaction_test
import (
"encoding/json"
"math"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) {
secrets := []string{"2", `quoted"\key`, "overlap", "overlap-long", "REDACTED"}
r := redaction.New(secrets)
original := map[string]any{"number": 2, "large": json.Number("9007199254740993"), "bool": true, "text": `echo:2 quoted"\key overlap-long overlap`}
raw, err := r.JSON(original)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
if decoded["number"] != float64(2) || decoded["bool"] != true || !strings.Contains(string(raw), "9007199254740993") {
t.Fatalf("changed scalar meanings: %s", raw)
}
if decoded["text"] != "echo:[REDACTED] [REDACTED] [REDACTED] [REDACTED]" {
t.Fatalf("redaction: %s", raw)
}
if original["text"] != `echo:2 quoted"\key overlap-long overlap` {
t.Fatal("mutated domain data")
}
if raw, err := r.JSON(math.NaN()); err == nil || raw != nil {
t.Fatalf("unsafe fallback: %s %v", raw, err)
}
if raw, err := r.JSON(map[string]string{"overlap": "first", "overlap-long": "second"}); err == nil || raw != nil {
t.Fatalf("colliding redacted keys: %s %v", raw, err)
}
nested, _ := json.Marshal(map[string]string{"reason": `quoted"\key`})
raw, err = r.JSON(map[string]string{"final_content": string(nested)})
if err != nil || strings.Contains(string(raw), "key") {
t.Fatalf("nested content: %s %v", raw, err)
}
yaml, err := r.YAML([]byte("version: 2\nenabled: true\nmessage: echo:2\n"))
if err != nil || !strings.Contains(string(yaml), "version: 2") || !strings.Contains(string(yaml), "echo:[REDACTED]") {
t.Fatalf("YAML: %s %v", yaml, err)
}
}
func TestDebugWritersRedactSourceAndNormalizedSnapshots(t *testing.T) {
dir := t.TempDir()
run, err := diagnostics.NewRunDirectory(dir, "run-test", "2")
if err != nil {
t.Fatal(err)
}
if err := run.WriteSourceTranscript(&schema.SourceTranscript{}, []byte(`{"segments":[{"text":"echo:2","id":2}]}`)); err != nil {
t.Fatal(err)
}
if err := run.WriteNormalizedTranscript(&schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "echo:2"}}}); err != nil {
t.Fatal(err)
}
for _, path := range []string{filepath.Join(run.Path(), diagnostics.ArtifactSourceTranscript), filepath.Join(run.Path(), diagnostics.ArtifactNormalizedTranscript)} {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "echo:2") || !strings.Contains(string(raw), "echo:[REDACTED]") || !strings.Contains(string(raw), `"id": 2`) {
t.Fatalf("snapshot: %s", raw)
}
}
if err := run.WriteJSONArtifact("invalid.json", map[string]any{"secret": "echo:2", "invalid": math.NaN()}); err == nil {
t.Fatal("expected serialization failure")
}
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !os.IsNotExist(err) {
t.Fatalf("failed artifact persisted: %v", err)
}
}

View File

@@ -5,8 +5,6 @@ import (
"html" "html"
"sort" "sort"
"strings" "strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
) )
// CompletionLine is the brief successful CLI summary. It uses the same counts // CompletionLine is the brief successful CLI summary. It uses the same counts
@@ -24,12 +22,10 @@ func count(value *int) int {
} }
// Markdown renders a fixed-layout human report from one processing snapshot. // Markdown renders a fixed-layout human report from one processing snapshot.
func Markdown(snapshot Snapshot, secrets ...string) []byte { func Markdown(snapshot Snapshot) []byte {
r := redaction.New(secrets)
safe := func(value string) string { return escapeMarkdown(r.String(value)) }
var b strings.Builder var b strings.Builder
fmt.Fprintf(&b, "# Audita report: %s\n\n", safe(snapshot.RunID)) fmt.Fprintf(&b, "# Audita report: %s\n\n", escapeMarkdown(snapshot.RunID))
fmt.Fprintf(&b, "Status: %s\n\n", safe(snapshot.Status)) fmt.Fprintf(&b, "Status: %s\n\n", escapeMarkdown(snapshot.Status))
fmt.Fprintf(&b, "Corrections applied: %d \n", count(snapshot.Summary.CorrectionsApplied)) fmt.Fprintf(&b, "Corrections applied: %d \n", count(snapshot.Summary.CorrectionsApplied))
fmt.Fprintf(&b, "Work units: %s total; %s completed; %s interrupted; %s not started \n", fmt.Fprintf(&b, "Work units: %s total; %s completed; %s interrupted; %s not started \n",
showCount(snapshot.Summary.WorkUnitsTotal), showCount(snapshot.Summary.WorkUnitsCompleted), showCount(snapshot.Summary.WorkUnitsTotal), showCount(snapshot.Summary.WorkUnitsCompleted),
@@ -38,20 +34,20 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintf(&b, "Recoveries: %s of %s model operations \n", showCount(snapshot.Summary.Recoveries), showCount(snapshot.Summary.LLMOperations)) fmt.Fprintf(&b, "Recoveries: %s of %s model operations \n", showCount(snapshot.Summary.Recoveries), showCount(snapshot.Summary.LLMOperations))
fmt.Fprintf(&b, "Proposal responses: %s; with filtering: %s\n\n", fmt.Fprintf(&b, "Proposal responses: %s; with filtering: %s\n\n",
showCount(snapshot.Summary.ProposalResponses), showCount(snapshot.Summary.ProposalResponsesWithFiltering)) showCount(snapshot.Summary.ProposalResponses), showCount(snapshot.Summary.ProposalResponsesWithFiltering))
fmt.Fprintf(&b, "Transcript input: %s \n", safe(snapshot.TranscriptPath)) fmt.Fprintf(&b, "Transcript input: %s \n", escapeMarkdown(snapshot.TranscriptPath))
output := snapshot.OutputPath output := snapshot.OutputPath
if output == "" { if output == "" {
output = "stdout" output = "stdout"
} }
fmt.Fprintf(&b, "Transcript output: %s\n\n", safe(output)) fmt.Fprintf(&b, "Transcript output: %s\n\n", escapeMarkdown(output))
fmt.Fprintln(&b, "## Work-unit errors") fmt.Fprintln(&b, "## Work-unit errors")
fmt.Fprintln(&b) fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, false) writeMatrix(&b, snapshot, false)
fmt.Fprintln(&b) fmt.Fprintln(&b)
fmt.Fprintln(&b, "## Recoveries") fmt.Fprintln(&b, "## Recoveries")
fmt.Fprintln(&b) fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, true) writeMatrix(&b, snapshot, true)
fmt.Fprintln(&b) fmt.Fprintln(&b)
if len(snapshot.RepairOutcomes) > 0 { if len(snapshot.RepairOutcomes) > 0 {
fmt.Fprintln(&b, "## Correction repair") fmt.Fprintln(&b, "## Correction repair")
@@ -59,14 +55,14 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |") fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |")
fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |") fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |")
for _, row := range snapshot.RepairOutcomes { for _, row := range snapshot.RepairOutcomes {
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome)) fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, escapeMarkdown(row.Outcome))
} }
fmt.Fprintln(&b) fmt.Fprintln(&b)
} }
fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.") fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.")
fmt.Fprintln(&b) fmt.Fprintln(&b)
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe) writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors))
writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries), safe) writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries))
fmt.Fprintln(&b, "A recovery can have multiple reasons; reason counts need not sum to the recovery total.") fmt.Fprintln(&b, "A recovery can have multiple reasons; reason counts need not sum to the recovery total.")
fmt.Fprintln(&b) fmt.Fprintln(&b)
fmt.Fprintln(&b, "Details: `errors.yml` and `recoveries.yml`.") fmt.Fprintln(&b, "Details: `errors.yml` and `recoveries.yml`.")
@@ -80,7 +76,7 @@ func showCount(value *int) string {
return fmt.Sprint(*value) return fmt.Sprint(*value)
} }
func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redactor, recovery bool) { func writeMatrix(b *strings.Builder, snapshot Snapshot, recovery bool) {
if len(snapshot.StageNames) == 0 || len(snapshot.WorkUnits) == 0 { if len(snapshot.StageNames) == 0 || len(snapshot.WorkUnits) == 0 {
fmt.Fprintln(b, "No work units.") fmt.Fprintln(b, "No work units.")
return return
@@ -88,7 +84,7 @@ func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redac
columns := snapshot.StageNames columns := snapshot.StageNames
fmt.Fprint(b, "| Chunk |") fmt.Fprint(b, "| Chunk |")
for _, name := range columns { for _, name := range columns {
fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(redactor.String(name)))) fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(name)))
} }
fmt.Fprint(b, "\n| --- |") fmt.Fprint(b, "\n| --- |")
for range columns { for range columns {
@@ -137,7 +133,7 @@ var stageCellEscaper = strings.NewReplacer(
"://", "&#58;//", "www.", "www&#46;", "@", "&#64;", "://", "&#58;//", "www.", "www&#46;", "@", "&#64;",
) )
func writeReasons(b *strings.Builder, title string, counts map[string]int, safe func(string) string) { func writeReasons(b *strings.Builder, title string, counts map[string]int) {
fmt.Fprintf(b, "### %s\n\n", title) fmt.Fprintf(b, "### %s\n\n", title)
if len(counts) == 0 { if len(counts) == 0 {
fmt.Fprintln(b, "None.") fmt.Fprintln(b, "None.")
@@ -150,7 +146,7 @@ func writeReasons(b *strings.Builder, title string, counts map[string]int, safe
} }
sort.Strings(keys) sort.Strings(keys)
for _, key := range keys { for _, key := range keys {
fmt.Fprintf(b, "- `%s`: %d\n", safe(key), counts[key]) fmt.Fprintf(b, "- `%s`: %d\n", escapeMarkdown(key), counts[key])
} }
fmt.Fprintln(b) fmt.Fprintln(b)
} }

View File

@@ -28,7 +28,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
Errors: []ErrorRecord{{Scope: "work_unit", Failures: []Failure{{Reason: "transport_failed"}, {Reason: "transport_failed"}}}}, Errors: []ErrorRecord{{Scope: "work_unit", Failures: []Failure{{Reason: "transport_failed"}, {Reason: "transport_failed"}}}},
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}}, Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
} }
got := Markdown(snapshot, "secret") got := Markdown(snapshot)
text := string(got) text := string(got)
// Compare only complete table rows, preserving order and states while leaving // Compare only complete table rows, preserving order and states while leaving
// prose and spacing outside the fixed matrices free to change. // prose and spacing outside the fixed matrices free to change.
@@ -45,7 +45,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) { if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
t.Fatalf("matrices: got %v; want %v", rows, wantRows) t.Fatalf("matrices: got %v; want %v", rows, wantRows)
} }
for _, fragment := range []string{"in\\|put\\`[REDACTED]\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} { for _, fragment := range []string{"in\\|put\\`secret\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
if !strings.Contains(text, fragment) { if !strings.Contains(text, fragment) {
t.Fatalf("missing %q in:\n%s", fragment, text) t.Fatalf("missing %q in:\n%s", fragment, text)
} }
@@ -69,9 +69,6 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
t.Fatalf("%s summary: got %v; want %v", label, got, want) t.Fatalf("%s summary: got %v; want %v", label, got, want)
} }
} }
if strings.Contains(text, "secret") {
t.Fatal("secret leaked")
}
var decoded struct { var decoded struct {
Summary Summary `yaml:"summary"` Summary Summary `yaml:"summary"`
} }
@@ -88,12 +85,12 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
} }
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) { func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret")) text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}))
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") { if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
t.Fatal(text) t.Fatal(text)
} }
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret")) text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}))
if !strings.Contains(text, "bad\\|label\\`<br>\\[REDACTED\\]") { if !strings.Contains(text, "bad\\|label\\`<br>secret") {
t.Fatal(text) t.Fatal(text)
} }
} }

View File

@@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"time" "time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -178,17 +177,13 @@ func completionTime(t *time.Time) *string {
return &value return &value
} }
// YAML redacts only string nodes after typed serialization. // YAML encodes the report document.
func YAML(document any, secrets ...string) ([]byte, error) { func YAML(document any) ([]byte, error) {
raw, err := yaml.Marshal(document) raw, err := yaml.Marshal(document)
if err != nil { if err != nil {
return nil, fmt.Errorf("encode report YAML: %w", err) return nil, fmt.Errorf("encode report YAML: %w", err)
} }
safe, err := redaction.New(secrets).YAML(raw) return raw, nil
if err != nil {
return nil, fmt.Errorf("redact report YAML: %w", err)
}
return safe, nil
} }
// StageLabel identifies a configured entry by position, module, and optional ID. // StageLabel identifies a configured entry by position, module, and optional ID.

View File

@@ -8,7 +8,7 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) { func TestYAMLPreservesScalarTypes(t *testing.T) {
zero, one := 0, 1 zero, one := 0, 1
falseValue := false falseValue := false
snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC), snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC),
@@ -16,18 +16,15 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}}, Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}},
Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}} Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}}
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} { for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
payload, err := YAML(document, `secret\"key`, "1") payload, err := YAML(document)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if strings.Contains(string(payload), `secret\"key`) {
t.Fatalf("secret leaked: %s", payload)
}
var decoded map[string]any var decoded map[string]any
if err := yaml.Unmarshal(payload, &decoded); err != nil { if err := yaml.Unmarshal(payload, &decoded); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" { if decoded["run_id"] != snapshot.RunID || decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
t.Fatalf("wrong envelope types: %#v", decoded) t.Fatalf("wrong envelope types: %#v", decoded)
} }
summary := decoded["summary"].(map[string]any) summary := decoded["summary"].(map[string]any)
@@ -35,7 +32,7 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
t.Fatalf("changed count types: %#v", summary) t.Fatalf("changed count types: %#v", summary)
} }
} }
if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") { if payload, err := YAML(snapshot.RecoveriesDocument()); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
t.Fatalf("boolean changed: %s %v", payload, err) t.Fatalf("boolean changed: %s %v", payload, err)
} }
} }

View File

@@ -25,13 +25,11 @@ type CallHooks struct {
} }
type RoleClient struct { type RoleClient struct {
runtime *Runtime runtime *Runtime
role string role string
profileID string profileID string
settings config.LLM settings config.LLM
hooks CallHooks hooks CallHooks
credentialMu sync.Mutex
credentialNames map[string]struct{}
} }
var _ structuredoutput.Client = (*RoleClient)(nil) var _ structuredoutput.Client = (*RoleClient)(nil)
@@ -152,14 +150,6 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var details pk.PreparedRun var details pk.PreparedRun
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) { capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
details = prepared details = prepared
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
c.credentialNames = make(map[string]struct{})
}
c.credentialNames[name] = struct{}{}
c.credentialMu.Unlock()
}
response.Prompt = promptMetadata(details) response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model response.Model = details.EffectiveModelParams.Model
@@ -311,17 +301,6 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)} return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
} }
// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials.
func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
c.credentialMu.Lock()
names := make([]string, 0, len(c.credentialNames))
for name := range c.credentialNames {
names = append(names, name)
}
c.credentialMu.Unlock()
return c.runtime.ConfiguredSecrets(ctx, names...)
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message { func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages)) result := make([]structuredoutput.Message, len(messages))
for i, message := range messages { for i, message := range messages {

View File

@@ -1,13 +1,12 @@
package llm package llm
import ( import (
"encoding/json"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
"strings" "strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
) )
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
@@ -23,13 +22,12 @@ type InteractionArtifacts struct {
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts. // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
type DiagnosticsWriter struct { type DiagnosticsWriter struct {
dir string dir string
redactor redaction.Redactor
} }
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir. // NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter { func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)} return &DiagnosticsWriter{dir: dir}
} }
// WriteInteraction writes request/response/error artifacts for a generic LLM // WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,12 +80,12 @@ func sanitizeStageName(stage string) string {
} }
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error { func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
redacted, err := w.redactor.JSON(payload) encoded, err := json.Marshal(payload)
if err != nil { if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err) return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
} }
redacted = append(redacted, '\n') encoded = append(encoded, '\n')
if err := os.WriteFile(path, redacted, 0o644); err != nil { if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err) return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
} }
return nil return nil

View File

@@ -4,13 +4,12 @@ import (
"encoding/json" "encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
) )
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) { func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
writer := NewDiagnosticsWriter(dir, nil) writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction( artifacts, err := writer.WriteInteraction(
"validation:batch-1", "validation:batch-1",
@@ -31,45 +30,6 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
assertValidJSONFile(t, artifacts.ResponsePayloadPath) assertValidJSONFile(t, artifacts.ResponsePayloadPath)
} }
func TestDiagnosticsWriterRedactsSecrets(t *testing.T) {
dir := t.TempDir()
secret := "top-secret-token"
writer := NewDiagnosticsWriter(dir, []string{secret})
artifacts, err := writer.WriteInteraction(
"module glossary / request",
map[string]any{"authorization": "Bearer " + secret},
map[string]any{"api_key": secret, "nested": map[string]any{"value": secret}},
map[string]any{"echo": secret},
map[string]any{"error": "failed with " + secret},
)
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
for _, path := range []string{
artifacts.RequestMetadataPath,
artifacts.RequestPayloadPath,
artifacts.ResponsePayloadPath,
artifacts.ErrorPayloadPath,
} {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read %s: %v", path, readErr)
}
content := string(raw)
if strings.Contains(content, secret) {
t.Fatalf("secret leaked in %s: %s", filepath.Base(path), content)
}
if !strings.Contains(content, "[REDACTED]") {
t.Fatalf("redaction marker missing in %s: %s", filepath.Base(path), content)
}
if !json.Valid(raw) {
t.Fatalf("invalid json in %s", filepath.Base(path))
}
}
}
func assertValidJSONFile(t *testing.T, path string) { func assertValidJSONFile(t *testing.T, path string) {
t.Helper() t.Helper()
raw, err := os.ReadFile(path) raw, err := os.ReadFile(path)

View File

@@ -9,7 +9,6 @@ import (
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -26,7 +25,7 @@ type transportFunc func(*http.Request) (*http.Response, error)
func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) { func TestRoleClientUsesSelectedNativeCredential(t *testing.T) {
t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ") t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ")
dir := t.TempDir() dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil { if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
@@ -41,7 +40,7 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
calls := 0 calls := 0
providerErr := errors.New("provider echoed synthetic-secret-value") providerErr := errors.New("transport stopped")
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) { httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
calls++ calls++
if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" { if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" {
@@ -62,10 +61,6 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
if err == nil || calls != 1 { if err == nil || calls != 1 {
t.Fatalf("error = %v, calls = %d", err, calls) t.Fatalf("error = %v, calls = %d", err, calls)
} }
masked := runtime.RedactError(context.Background(), providerErr)
if strings.Contains(masked, "synthetic-secret-value") {
t.Fatal("credential not redacted")
}
} }
func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) { func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
@@ -282,10 +277,4 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
} }
}) })
} }
secrets := runtime.ConfiguredSecrets(context.Background())
for _, value := range []string{"backend-secret-value", "profile-secret-value"} {
if !slices.Contains(secrets, value) {
t.Fatalf("missing credential masking value")
}
}
} }

View File

@@ -8,7 +8,6 @@ import (
"net/http" "net/http"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,12 +19,10 @@ import (
// Runtime owns the captured engine shared by all processing clients. // Runtime owns the captured engine shared by all processing clients.
type Runtime struct { type Runtime struct {
Config config.Config Config config.Config
Engine *pk.Engine Engine *pk.Engine
SourcePaths []string SourcePaths []string
nextCallID atomic.Uint64 nextCallID atomic.Uint64
credentialSourcesOnce sync.Once
credentialEnvNames map[string]struct{}
} }
type AssetError struct{ Err error } type AssetError struct{ Err error }

View File

@@ -1,78 +0,0 @@
package llm
import (
"context"
"os"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
// ConfiguredSecrets obtains only values referenced by captured credential metadata.
// It never selects a credential or treats an absent value as an error.
func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string {
if r == nil || r.Engine == nil {
return nil
}
r.credentialSourcesOnce.Do(func() {
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
})
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
read := func(name string) {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
for name := range r.credentialEnvNames {
read(name)
}
if len(additionalEnvNames) > 0 {
seen := make(map[string]struct{}, len(additionalEnvNames))
for _, name := range additionalEnvNames {
if _, known := r.credentialEnvNames[name]; known {
continue
}
if _, duplicate := seen[name]; name == "" || duplicate {
continue
}
seen[name] = struct{}{}
read(name)
}
}
return secrets
}
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
names := make(map[string]struct{})
add := func(name *string) {
if name != nil && *name != "" {
names[*name] = struct{}{}
}
}
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
for _, id := range ids {
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
add(profile.APIKeyEnv)
}
}
}
if ids, err := r.Engine.ListBackends(ctx); err == nil {
for _, id := range ids {
if backend, err := r.Engine.InspectBackend(ctx, id); err == nil {
add(backend.APIKeyEnv)
}
}
}
return names
}
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {
if err == nil {
return ""
}
return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error())
}

View File

@@ -1,52 +0,0 @@
package llm
import (
"context"
"os"
"path/filepath"
"slices"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
t.Setenv("SELECTED_KEY", "rotated-secret")
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
slices.Sort(values)
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
}
}

View File

@@ -28,7 +28,6 @@ type DebugInput struct {
RunID string RunID string
Metadata diagnostics.InvocationMetadata Metadata diagnostics.InvocationMetadata
EffectiveConfig []byte EffectiveConfig []byte
Secrets []string
} }
type Prepared struct { type Prepared struct {
@@ -85,7 +84,7 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
} }
result := &Prepared{Output: session} result := &Prepared{Output: session}
if input.Debug != nil { if input.Debug != nil {
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig, input.Debug.Secrets...) result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig)
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, cancelOutput(err) return nil, cancelOutput(err)

View File

@@ -130,17 +130,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if req.DiagnosticsWriter != nil { if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" { } else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{ writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter( writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
} }
} }

View File

@@ -19,7 +19,6 @@ import (
) )
type fakeStructuredClient struct { type fakeStructuredClient struct {
secrets []string
responses []StructuredCorrectionSet responses []StructuredCorrectionSet
err error err error
completion contracts.StructuredCompletionResponse completion contracts.StructuredCompletionResponse
@@ -50,8 +49,6 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
return response, nil return response, nil
} }
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
type captureDiagnosticsWriter struct { type captureDiagnosticsWriter struct {
err error err error
lastStage string lastStage string
@@ -311,21 +308,16 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
} }
} }
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) { func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
primarySecret := "proposal-primary-secret"
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{ client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{ responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}}, {Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}},
}, },
} }
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
client.secrets = []string{primarySecret, validationSecret}
req := defaultRequest(t) req := defaultRequest(t)
req.LLMClient = client req.LLMClient = client
req.DiagnosticsDir = t.TempDir() req.DiagnosticsDir = t.TempDir()
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret} req.Variables = map[string]string{"SectionJSON": "section text"}
got, err := GenerateCandidates(context.Background(), req) got, err := GenerateCandidates(context.Background(), req)
if err != nil { if err != nil {
@@ -340,11 +332,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil { if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr) t.Fatalf("read artifact %q: %v", path, readErr)
} }
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { if !json.Valid(raw) {
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw)) t.Fatalf("invalid JSON artifact %q: %s", path, raw)
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
} }
} }

View File

@@ -7,8 +7,6 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} }
} }
current := original current := original
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding { findings := repairFindings(original.ProposalIndex, rejected)
findings := repairFindings(original.ProposalIndex, rejected)
redactor := redaction.New(repairSecrets(ctx, input))
for i := range findings {
findings[i].Explanation = redactor.String(findings[i].Explanation)
}
return findings
}
findings := safeFindings(rejected)
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator) coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
if !ok { if !ok {
return finish(fmt.Errorf("correction repair coordinator is unavailable")) return finish(fmt.Errorf("correction repair coordinator is unavailable"))
@@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if err != nil { if err != nil {
failure = map[string]string{"error": err.Error()} failure = map[string]string{"error": err.Error()}
} }
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input)) writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure) artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
if writeErr == nil { if writeErr == nil {
record.Artifacts = artifacts record.Artifacts = artifacts
@@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return nil, nil return nil, nil
} }
if actionable(original.ProposalIndex, validated.rejected) { if actionable(original.ProposalIndex, validated.rejected) {
findings = safeFindings(validated.rejected) findings = repairFindings(original.ProposalIndex, validated.rejected)
return buildRequest() return buildRequest()
} }
branch.Outcome = "unavailable" branch.Outcome = "unavailable"
@@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
} }
return finish(nil) return finish(nil)
} }
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
var secrets []string
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
if source, ok := client.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
}
}
return secrets
}

View File

@@ -31,7 +31,6 @@ type repairClient struct {
outputs []string outputs []string
calls []structuredoutput.Request calls []structuredoutput.Request
err error err error
secrets []string
} }
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) { func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
} }
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) { func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}} client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
candidates := []proposals.EnrichedCorrectionProposal{ candidates := []proposals.EnrichedCorrectionProposal{
@@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
} }
} }
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) { func TestRepairArtifactsAreUnique(t *testing.T) {
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}} client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
candidates := []proposals.EnrichedCorrectionProposal{ candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}, {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
@@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
if a == "" || a == b { if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b) t.Fatalf("artifact paths: %q %q", a, b)
} }
for _, path := range []string{a, b} { for i, path := range []string{a, b} {
data, err := os.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") { if !json.Valid(data) {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data) t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
}
var request proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
t.Fatal(err)
}
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
t.Fatalf("repair request for proposal %d: %+v", i, request)
} }
} }
} }
@@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
} }
} }
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
}
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "validation-secret") {
t.Fatalf("credential in artifact: %s", data)
}
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam. // rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator { func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
dir := t.TempDir() dir := t.TempDir()
for name, content := range map[string]string{ for name, content := range map[string]string{
"config.yml": "version: 3\n", "config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n", "prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
} { } {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err) t.Fatal(err)

View File

@@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) { if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil { if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)} interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
if source, ok := input.ValidationLLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
}
} }
diagnosticsWriter = interactionWriter diagnosticsWriter = interactionWriter
} }
@@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
} }
type llmDiagnosticsWriterAdapter struct { type llmDiagnosticsWriterAdapter struct {
dir string dir string
secrets func() []string
} }
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
var secrets []string art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if a.secrets != nil {
secrets = a.secrets()
}
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil { if err != nil {
return validators.InteractionArtifacts{}, err return validators.InteractionArtifacts{}, err
} }

View File

@@ -2,6 +2,7 @@ package runner
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
@@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
func ptrConfig(c config.Config) *config.Config { return &c } func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct { type fakeStructuredClient struct {
responses []validators.LLMValidationResponse responses []validators.LLMValidationResponse
secrets []string err error
preparedSecret string calls int
err error
calls int
} }
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx _ = ctx
_ = req _ = req
f.calls++ f.calls++
if f.preparedSecret != "" {
f.secrets = append(f.secrets, f.preparedSecret)
f.preparedSecret = ""
}
if f.err != nil { if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err return contracts.StructuredCompletionResponse{}, f.err
} }
@@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
} }
} }
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
primarySecret := "runner-primary-secret" client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
cfg := runnerTestConfig() cfg := runnerTestConfig()
diagDir := t.TempDir() diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
}}) }})
out, err := r.Run(context.Background(), RunInput{ out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}}, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir, ValidationDiagnosticsDir: diagDir,
}) })
@@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil { if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr) t.Fatalf("read diagnostic %q: %v", path, readErr)
} }
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) { if !json.Valid(raw) {
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw)) t.Fatalf("invalid diagnostic %q: %s", path, raw)
} }
} }
@@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil { if readErr != nil {
t.Fatalf("read decision diagnostic: %v", readErr) t.Fatalf("read decision diagnostic: %v", readErr)
} }
if !strings.Contains(string(raw), "[REDACTED]") { if !strings.Contains(string(raw), "supported by context") {
t.Fatalf("expected redaction marker in diagnostics") t.Fatalf("missing decision evidence: %s", raw)
} }
} }