Remove application credential substitution

This commit is contained in:
2026-11-16 03:37:29 +00:00
parent e34672f7fe
commit ffe0e3491e
37 changed files with 217 additions and 729 deletions

View File

@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
@@ -118,14 +119,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
_, _ = io.WriteString(w, `{"error":{"message":"provider unavailable"}}`)
return
}
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
if p.mode == "invalid-credential-echo" {
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
content = string(raw)
if p.mode == "malformed-response" {
content = `{"corrections": [`
}
if p.mode == "repair-filter-later-failure" && !validation {
if proposalCall == 1 {
@@ -246,9 +245,6 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
t.Helper()
dir := t.TempDir()
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
if mode == "invalid-credential-echo" {
p.secret = " synthetic-echo-credential\t "
}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
@@ -957,40 +953,37 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
t.Fatalf("model failure report: %+v", document)
}
if strings.Contains(stderr, f.provider.secret) {
t.Fatal("secret leaked in stderr")
}
}
func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) {
f := newProcessFixture(t, "invalid-credential-echo", "[grammar]")
effective := strings.TrimSpace(f.provider.secret)
func TestSubprocessCapturesMalformedProviderResponse(t *testing.T) {
f := newProcessFixture(t, "malformed-response", "[grammar]")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
}
if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) {
t.Fatal("credential escaped redaction in command output")
}
responseCaptured := false
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() {
if err != nil {
return err
}
if entry.IsDir() || !strings.HasSuffix(entry.Name(), "-response-payload.json") {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
if bytes.Contains(data, []byte(effective)) {
t.Errorf("credential escaped redaction in %s", entry.Name())
var response struct {
FinalContent string `json:"final_content"`
}
if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) {
responseCaptured = true
if err := json.Unmarshal(data, &response); err != nil {
return err
}
responseCaptured = responseCaptured || response.FinalContent == `{"corrections": [`
return nil
})
if err != nil || !responseCaptured {
t.Fatalf("missing redacted provider echo: %v", err)
t.Fatalf("missing malformed provider response: %v", err)
}
}
@@ -1246,27 +1239,16 @@ func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
}
}
func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
func TestSubprocessAnnouncesActualOutputPaths(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(fmt.Sprint(fail), func(t *testing.T) {
f := newProcessFixture(t, "normal", "[]")
selected, unused := "selected-path-secret", "unused-path-secret"
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
reportDir := filepath.Join(f.dir, "reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
f.output = filepath.Join(reportDir, "out.json")
debugParent := filepath.Join(f.dir, selected)
debugParent := filepath.Join(f.dir, "diagnostics")
if fail {
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
t.Fatal(err)
@@ -1276,11 +1258,8 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
t.Fatalf("credential in stderr: %q", stderr)
}
if !fail && !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
if !fail && (!strings.Contains(stderr, filepath.Join(reportDir, "report.md")) || !strings.Contains(stderr, debugParent)) {
t.Fatalf("missing destination announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err)

View File

@@ -11,11 +11,10 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
applicable. Preserve the precedence defined in the
[configuration contract](../config.md#loading-and-composition), including the
distinct validation and effective-config commands.
3. Update redaction when secrets are involved.
4. Review configuration tests and related CLI tests for precedence, validation,
3. Review configuration tests and related CLI tests for precedence, validation,
and source-specific behavior. Add or adjust tests for the changed contract
according to the [testing policy](../policy/testing.md).
5. Update the configuration reference and affected [examples](../../examples/).
4. Update the configuration reference and affected [examples](../../examples/).
## Changing CLI Behavior
@@ -51,8 +50,7 @@ Select checks according to the [testing policy](../policy/testing.md):
- Process/output changes: verify stdout, stderr, exit, output-file, and report
behavior against the [CLI contract](../cli.md) and
[subprocess guidance](../integrations/subprocess.md), including failures.
- Report or diagnostics changes: review [report assembly tests and redaction
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
- Report or diagnostics changes: review [report assembly tests](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
and `internal/core/schema` tests, including supported formats and rejection of
unsupported schemas.

View File

@@ -12,8 +12,7 @@ work-unit errors, separate run errors, and one recovery per successful logical
operation. Reports order stages, sections, validators, and batches by semantic
identity, never concurrent completion time. The core reporting package owns
Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI
completion formatter. YAML redaction walks serialized string nodes so numeric
and boolean values retain their types. Ordinary reports omit raw causes,
completion formatter. Reports encode the typed documents directly to YAML. Ordinary reports omit raw causes,
transcript text, complete decisions, and token/timing payloads.
Validation operations with unavailable decisions report only affected requested
indices, even when other candidates in the batch continue. Ignored extra
@@ -34,17 +33,16 @@ pass; a published transcript is never erased by a later report failure.
Debug is independent. The shared `--debug` flag enables a unique retained child
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
when debug is off. With debug enabled, it captures source and normalized
snapshots, invocation, effective settings, correction ledger, LLM execution
observations, and model interactions. Candidate capture includes proposals later
when debug is off. With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized
transcript snapshots. It also captures invocation metadata, effective settings
rendered by ConfigKit, the correction ledger, LLM execution observations, and
model interactions. Candidate capture includes proposals later
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable; their retained content
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
replacements and distinguishes unavailable candidates from interrupted work.
attempt, validator, and batch coordinates as applicable. Ledger mapping preserves empty proposed and applied replacements and distinguishes unavailable candidates from interrupted work.
A concurrency-safe diagnostic callback reports optional write failures without
aborting processing. Diagnostic redaction uses credential source metadata; debug
artifacts can still contain sensitive transcript and prompt content. Execution
aborting processing. Debug artifacts can contain sensitive transcript and
prompt content. Execution
position and optional operator ID are carried separately from generated safe
artifact path coordinates.

View File

@@ -15,9 +15,7 @@ The operation budget includes preparation, admission, retries, and structural
repair. `ErrOperationTimeout` remains recoverable even when wrapped by a
preparation error. Parent cancellation remains distinct. The prepared notification
provides the actual request snapshot; failures before notification have no snapshot.
PromptKit owns backend and credential selection. The adapter uses captured
credential source names for artifact redaction without inspecting unrelated
environment values. See the [integration contract](../integrations/openai-compatible-llm.md)
PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md)
for failure behavior and the [configuration reference](../config.md) for sources.
When correction repair is enabled, the proposal call captures a native immutable

View File

@@ -67,7 +67,7 @@ artifact lifecycle.
## Component Responsibilities
Core components implement configuration defaults/loading/validation/redaction,
Core components implement configuration defaults, loading, and validation,
transcript schemas and derived glossary views, normalization and chunking, output encoding,
diagnostics artifact helpers, and public report structures.

View File

@@ -121,8 +121,7 @@ provider attempts. With debug enabled, utilization remains grouped by backend
capacity and usage fields retain known/complete flags. Debug runs write
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
`config-origins.yaml`, `utilization-diagnostics.json`, and
`llm-execution.json`. Request and response artifacts remain redacted debugging
material. See [operations](../operations.md#debug-evidence) for placement
`llm-execution.json`. Request and response artifacts are debugging material. See [operations](../operations.md#debug-evidence) for placement
and paths, [configuration](../config.md) for all settings, and the
[CLI reference](../cli.md) for current flags.

View File

@@ -48,11 +48,9 @@ existence as proof of success.
The shared `--debug` flag enables retained artifacts in a run directory beneath
`diagnostics.work_dir`; selecting a work directory alone does not enable capture.
Optional debug preparation or writes can produce redacted warnings without
Optional debug preparation or writes can produce warnings without
changing correction decisions or the required output result. Debug material can
include transcript content, prompts, responses, and correction decisions. It is
not anonymized; protect it according to the input's sensitivity. Configured
credential values are redacted at the diagnostic boundary. Repair interactions
include transcript content, prompts, responses, and correction decisions. Protect this material according to the sensitivity of the inputs and responses. Repair interactions
and revalidation artifacts include proposal and attempt coordinates; the
correction ledger connects them to the original edit.

View File

@@ -111,6 +111,12 @@ Audit #3, #9, #31 (source bytes), #63–65. Use the
bytes, malformed response evidence, actual paths, and native auth using
synthetic data. Check feedback by decoding a production repair request.
**Complete.** Reports and debug artifacts encode typed data directly, source
snapshots preserve the acquired bytes, and repair requests carry the original
domain findings. Removed application credential discovery and substitution.
The CLI build and full offline suite passed with Go 1.27.3, including checks
for native authentication, malformed response capture, and destination paths.
## Stage 3 — Unify optional interaction capture
Audit #33 and #43.

View File

@@ -36,7 +36,7 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
}
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil {
return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err)
return commandkit.Failure(err.Error()).WithCause(err)
}
defer prepared.Prepared.Output.Close()
var warningsMu sync.Mutex
@@ -63,24 +63,24 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
warn(debug.WriteNormalizationSummary(summary))
}
if err := inv.Context.Err(); err != nil {
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err)
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, err)
}
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
if runErr != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr)
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
}
encoder, err := outputschema.Resolve(cfg.Output.Schema)
if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
}
if result.Transcript == nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
return finishProcess(inv, input, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
}
payload, err := encoder.Encoder(result.Transcript)
if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
}
return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil)
return finishProcess(inv, input, prepared, result.Snapshot, payload, nil)
}
// DebugPath returns the retained artifact directory, if its preparation succeeded.

View File

@@ -87,10 +87,6 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
return nil, err
}
started := time.Now().UTC()
var secrets []string
if cfg.Reporting.Enabled || input.Debug.Value {
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
}
var initial diagnostics.ReportBundle
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
if cfg.Reporting.Enabled {
@@ -98,7 +94,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
}
initial, err = renderReportBundle(snapshot, secrets...)
initial, err = renderReportBundle(snapshot)
if err != nil {
return nil, err
}
@@ -109,7 +105,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
if primary != nil {
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
}
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures))
return bundle
}
}
@@ -120,7 +116,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
if err := loaded.Configuration.WriteEffective(&effective); err != nil {
warnings = append(warnings, err)
}
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
}
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
@@ -147,14 +143,14 @@ func generateRunID() (string, error) {
return "run-" + hex.EncodeToString(random[:]), nil
}
func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) {
errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...)
func renderReportBundle(value reporting.Snapshot) (diagnostics.ReportBundle, error) {
errorsYAML, err := reporting.YAML(value.ErrorsDocument())
if err != nil {
return diagnostics.ReportBundle{}, err
}
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...)
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument())
if err != nil {
return diagnostics.ReportBundle{}, err
}
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value)}, nil
}

View File

@@ -147,7 +147,11 @@ pipeline:
if readErr != nil {
t.Fatal(readErr)
}
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) {
var compact bytes.Buffer
if err := json.Compact(&compact, raw); err != nil {
t.Fatalf("decode %s: %v", artifact, err)
}
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw)
}
}

View File

@@ -7,19 +7,16 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
commandkit "go.maximumdirect.net/commandkit"
)
// finishProcess delivers the transcript and reports before emitting success information.
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
func finishProcess(inv commandkit.Invocation, input processInput, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
reporter := inv.Reporter()
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
safe := redaction.New(secrets)
for _, warning := range prepared.Prepared.Warnings {
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
_ = reporter.Warn("Debug artifact: " + warning.Error())
}
completed := time.Now().UTC()
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
@@ -36,7 +33,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
if len(reportPaths) > 0 {
var err error
reports, err = renderReportBundle(snapshot, secrets...)
reports, err = renderReportBundle(snapshot)
if err != nil {
if primary == nil {
primary = err
@@ -44,7 +41,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
snapshot.Status = reporting.StatusFailed
}
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures))
if bundleErr != nil {
return reports
}
@@ -60,14 +57,14 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
}
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil {
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
return commandkit.Failure(result.Error()).WithCause(result)
}
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range reportPaths {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", path))
}
if prepared.Prepared.Debug != nil {
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path())))
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", prepared.Prepared.Debug.Path()))
}
return commandkit.Success()
}

View File

@@ -46,7 +46,7 @@ func TestPreparedDeliveryFollowsReportingMode(t *testing.T) {
if err != nil {
return commandkit.Failure(err.Error())
}
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {
@@ -97,7 +97,7 @@ func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) {
if err != nil {
return commandkit.Failure(err.Error())
}
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {
@@ -161,7 +161,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
t.Fatal(err)
}
cancel()
return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil)
return finishProcess(inv, input, prepared, result.Snapshot, []byte("new transcript"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {

View File

@@ -6,7 +6,6 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
@@ -25,7 +24,6 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
}
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
definitions := map[string]string{
"malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
@@ -38,9 +36,6 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatal("configured secret leaked")
}
})
}
}

View File

@@ -43,7 +43,7 @@ func (f FileFailure) Error() string { return fmt.Sprintf("%s %q: %v", f.Kind, f.
func (f FileFailure) Unwrap() error { return f.Err }
// OutputSession owns checked destinations and per-file publication. Filesystem
// failures remain typed so CLI can present safe, redacted diagnostics.
// failures remain typed so CLI can present contextual diagnostics.
type OutputSession struct {
transcript string
reports [3]string
@@ -250,8 +250,8 @@ func (s *OutputSession) Initialize(initial ReportBundle, failed func([]FileFailu
}
// PrepareDebug creates optional initial diagnostics and returns any nonfatal problems.
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte, secrets ...string) (*RunDirectory, []error) {
run, err := NewRunDirectory(workDir, runID, secrets...)
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte) (*RunDirectory, []error) {
run, err := NewRunDirectory(workDir, runID)
if err != nil {
return nil, []error{err}
}

View File

@@ -2,13 +2,17 @@ package diagnostics
import (
"bytes"
"encoding/json"
"errors"
"io"
"math"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
@@ -242,7 +246,7 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("unexpected parent: %v", err)
}
run, err := NewRunDirectory(parent, "run-123", "secret")
run, err := NewRunDirectory(parent, "run-123")
if err != nil {
t.Fatal(err)
}
@@ -252,11 +256,11 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
if _, err := NewRunDirectory(parent, "run-123"); err == nil {
t.Fatal("duplicate run ID accepted")
}
if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "secret"}); err != nil {
if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "edit"}); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger))
if err != nil || strings.Contains(string(contents), "secret") {
if err != nil || string(contents) != "{\"proposal\":\"edit\"}\n" {
t.Fatalf("debug artifact: %s %v", contents, err)
}
if _, err := os.Stat(run.Path()); err != nil {
@@ -264,6 +268,48 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
}
}
func TestDebugTranscriptSnapshotsPreserveSourceAndTypedViews(t *testing.T) {
run, err := NewRunDirectory(t.TempDir(), "run-transcripts")
if err != nil {
t.Fatal(err)
}
raw := []byte(" [\n {\"id\": 2, \"text\": \"literal\", \"speaker\": \"A\", \"start\": 0, \"end\": 1}\n ] ")
source, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
if err := run.WriteSourceTranscript(source, raw); err != nil {
t.Fatal(err)
}
normalized := &schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "normalized"}}}
if err := run.WriteNormalizedTranscript(normalized); err != nil {
t.Fatal(err)
}
acquired, err := os.ReadFile(filepath.Join(run.Path(), ArtifactSourceTranscript))
if err != nil || !bytes.Equal(acquired, raw) {
t.Fatalf("acquired source = %q, error = %v", acquired, err)
}
for name, want := range map[string]string{ArtifactParsedSourceTranscript: "literal", ArtifactNormalizedTranscript: "normalized"} {
payload, err := os.ReadFile(filepath.Join(run.Path(), name))
if err != nil {
t.Fatal(err)
}
var transcript schema.Transcript
if err := json.Unmarshal(payload, &transcript); err != nil {
t.Fatal(err)
}
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 2 || transcript.Segments[0].Text != want {
t.Fatalf("%s: %s", name, payload)
}
}
if err := run.WriteJSONArtifact("invalid.json", math.NaN()); err == nil {
t.Fatal("expected serialization failure")
}
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("failed artifact persisted: %v", err)
}
}
func TestSelectionAndAbortPreserveTranscript(t *testing.T) {
dir := t.TempDir()
output := filepath.Join(dir, "output.json")
@@ -363,7 +409,7 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 {
t.Fatalf("run=%v problems=%v", run, problems)
}
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("secret: sensitive\n"), "sensitive")
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("version: 3\n"))
if len(problems) != 0 {
t.Fatal(problems)
}
@@ -371,8 +417,8 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(contents), "sensitive") {
t.Fatal("debug artifact contains secret")
if string(contents) != "version: 3\n" {
t.Fatalf("changed effective configuration: %q", contents)
}
}

View File

@@ -9,13 +9,11 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
// RunDirectory represents a per-run diagnostics directory
type RunDirectory struct {
redactor redaction.Redactor
path string
createdAt time.Time
}
@@ -32,7 +30,7 @@ type InvocationMetadata struct {
}
// NewRunDirectory creates a new run directory under the configured work dir
func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, error) {
func NewRunDirectory(workDir, runID string) (*RunDirectory, error) {
if strings.TrimSpace(workDir) == "" {
return nil, fmt.Errorf("debug work directory is required")
}
@@ -53,7 +51,6 @@ func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, e
return &RunDirectory{
path: runPath,
redactor: redaction.New(secrets),
createdAt: time.Now().UTC(),
}, nil
}
@@ -77,7 +74,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
}
path := filepath.Join(r.path, ArtifactInvocationMetadata)
bytes, err := r.redactor.JSON(metadata)
bytes, err := json.Marshal(metadata)
if err != nil {
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
}
@@ -95,25 +92,14 @@ func (r *RunDirectory) WriteEffectiveConfig(effectiveYAML []byte) error {
// WriteSourceTranscript writes the source transcript artifact
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
// Preserve the source snapshot without copying unredacted input bytes.
var source any
decoder := json.NewDecoder(strings.NewReader(string(raw)))
decoder.UseNumber()
if err := decoder.Decode(&source); err != nil {
return fmt.Errorf("decode source snapshot: %w", err)
}
sourceBytes, err := r.redactor.JSON(source)
if err != nil {
return fmt.Errorf("redact source snapshot: %w", err)
}
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
if err := os.WriteFile(sourcePath, append(sourceBytes, '\n'), 0o644); err != nil {
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
return fmt.Errorf("failed to write source transcript: %w", err)
}
// Write parsed source for debugging
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
parsedBytes, err := r.redactor.JSON(transcript)
parsedBytes, err := json.Marshal(transcript)
if err != nil {
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
}
@@ -128,7 +114,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
// WriteNormalizedTranscript writes the normalized transcript artifact
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
bytes, err := r.redactor.JSON(transcript)
bytes, err := json.Marshal(transcript)
if err != nil {
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
}
@@ -141,7 +127,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
// WriteNormalizationSummary writes the normalization summary artifact
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
bytes, err := r.redactor.JSON(summary)
bytes, err := json.Marshal(summary)
if err != nil {
return fmt.Errorf("failed to marshal normalization summary: %w", err)
}
@@ -154,7 +140,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
artifactPath := filepath.Join(r.path, name)
bytes, err := r.redactor.JSON(payload)
bytes, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal %s: %w", name, err)
}
@@ -165,13 +151,8 @@ func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
return nil
}
// WriteYAMLArtifact writes a sanitized YAML diagnostic artifact.
// WriteYAMLArtifact writes the supplied YAML diagnostic artifact.
func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error {
var err error
payload, err = r.redactor.YAML(payload)
if err != nil {
return err
}
if err := os.WriteFile(filepath.Join(r.path, name), payload, 0o644); err != nil {
return fmt.Errorf("failed to write %s: %w", name, err)
}

View File

@@ -1,123 +0,0 @@
// Package redaction sanitizes diagnostic representations without modifying domain data.
package redaction
import (
"bytes"
"encoding/json"
"fmt"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
// Redactor replaces configured values in a single pass, longest matches first.
// Its zero value is usable and leaves strings unchanged.
type Redactor struct{ replacer *strings.Replacer }
func New(secrets []string) Redactor {
values := map[string]bool{}
for _, secret := range secrets {
if secret == "" {
continue
}
values[secret] = true
// Provider content may itself be serialized JSON inside a string field.
encoded, _ := json.Marshal(secret)
values[string(encoded[1:len(encoded)-1])] = true
}
keys := make([]string, 0, len(values))
for value := range values {
keys = append(keys, value)
}
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) == len(keys[j]) {
return keys[i] < keys[j]
}
return len(keys[i]) > len(keys[j])
})
pairs := make([]string, 0, 2*len(keys))
for _, key := range keys {
pairs = append(pairs, key, "[REDACTED]")
}
return Redactor{replacer: strings.NewReplacer(pairs...)}
}
func (r Redactor) String(value string) string {
if r.replacer == nil {
return value
}
return r.replacer.Replace(value)
}
// JSON serializes first, then redacts only decoded string values and keys.
// Numbers retain their exact representation. Failure never returns original data.
func (r Redactor) JSON(value any) ([]byte, error) {
raw, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize diagnostic: %s", r.String(err.Error()))
}
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
var decoded any
if err := decoder.Decode(&decoded); err != nil {
return nil, fmt.Errorf("decode diagnostic: %s", r.String(err.Error()))
}
safe, err := r.walk(decoded)
if err != nil {
return nil, err
}
return json.MarshalIndent(safe, "", " ")
}
func (r Redactor) walk(value any) (any, error) {
switch v := value.(type) {
case string:
return r.String(v), nil
case []any:
for i := range v {
var err error
v[i], err = r.walk(v[i])
if err != nil {
return nil, err
}
}
case map[string]any:
safe := make(map[string]any, len(v))
for key, item := range v {
safeKey := r.String(key)
if _, exists := safe[safeKey]; exists {
return nil, fmt.Errorf("redaction would produce duplicate object keys")
}
safeItem, err := r.walk(item)
if err != nil {
return nil, err
}
safe[safeKey] = safeItem
}
return safe, nil
}
return value, nil
}
// YAML preserves scalar types while sanitizing string nodes, including map keys.
func (r Redactor) YAML(raw []byte) ([]byte, error) {
var node yaml.Node
if err := yaml.Unmarshal(raw, &node); err != nil {
return nil, fmt.Errorf("decode diagnostic YAML: %s", r.String(err.Error()))
}
var visit func(*yaml.Node)
visit = func(n *yaml.Node) {
if n.Kind == yaml.ScalarNode && n.Tag == "!!str" {
n.Value = r.String(n.Value)
}
n.HeadComment = r.String(n.HeadComment)
n.LineComment = r.String(n.LineComment)
n.FootComment = r.String(n.FootComment)
for _, child := range n.Content {
visit(child)
}
}
visit(&node)
return yaml.Marshal(&node)
}

View File

@@ -1,81 +0,0 @@
package redaction_test
import (
"encoding/json"
"math"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) {
secrets := []string{"2", `quoted"\key`, "overlap", "overlap-long", "REDACTED"}
r := redaction.New(secrets)
original := map[string]any{"number": 2, "large": json.Number("9007199254740993"), "bool": true, "text": `echo:2 quoted"\key overlap-long overlap`}
raw, err := r.JSON(original)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
if decoded["number"] != float64(2) || decoded["bool"] != true || !strings.Contains(string(raw), "9007199254740993") {
t.Fatalf("changed scalar meanings: %s", raw)
}
if decoded["text"] != "echo:[REDACTED] [REDACTED] [REDACTED] [REDACTED]" {
t.Fatalf("redaction: %s", raw)
}
if original["text"] != `echo:2 quoted"\key overlap-long overlap` {
t.Fatal("mutated domain data")
}
if raw, err := r.JSON(math.NaN()); err == nil || raw != nil {
t.Fatalf("unsafe fallback: %s %v", raw, err)
}
if raw, err := r.JSON(map[string]string{"overlap": "first", "overlap-long": "second"}); err == nil || raw != nil {
t.Fatalf("colliding redacted keys: %s %v", raw, err)
}
nested, _ := json.Marshal(map[string]string{"reason": `quoted"\key`})
raw, err = r.JSON(map[string]string{"final_content": string(nested)})
if err != nil || strings.Contains(string(raw), "key") {
t.Fatalf("nested content: %s %v", raw, err)
}
yaml, err := r.YAML([]byte("version: 2\nenabled: true\nmessage: echo:2\n"))
if err != nil || !strings.Contains(string(yaml), "version: 2") || !strings.Contains(string(yaml), "echo:[REDACTED]") {
t.Fatalf("YAML: %s %v", yaml, err)
}
}
func TestDebugWritersRedactSourceAndNormalizedSnapshots(t *testing.T) {
dir := t.TempDir()
run, err := diagnostics.NewRunDirectory(dir, "run-test", "2")
if err != nil {
t.Fatal(err)
}
if err := run.WriteSourceTranscript(&schema.SourceTranscript{}, []byte(`{"segments":[{"text":"echo:2","id":2}]}`)); err != nil {
t.Fatal(err)
}
if err := run.WriteNormalizedTranscript(&schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "echo:2"}}}); err != nil {
t.Fatal(err)
}
for _, path := range []string{filepath.Join(run.Path(), diagnostics.ArtifactSourceTranscript), filepath.Join(run.Path(), diagnostics.ArtifactNormalizedTranscript)} {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "echo:2") || !strings.Contains(string(raw), "echo:[REDACTED]") || !strings.Contains(string(raw), `"id": 2`) {
t.Fatalf("snapshot: %s", raw)
}
}
if err := run.WriteJSONArtifact("invalid.json", map[string]any{"secret": "echo:2", "invalid": math.NaN()}); err == nil {
t.Fatal("expected serialization failure")
}
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !os.IsNotExist(err) {
t.Fatalf("failed artifact persisted: %v", err)
}
}

View File

@@ -5,8 +5,6 @@ import (
"html"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
// CompletionLine is the brief successful CLI summary. It uses the same counts
@@ -24,12 +22,10 @@ func count(value *int) int {
}
// Markdown renders a fixed-layout human report from one processing snapshot.
func Markdown(snapshot Snapshot, secrets ...string) []byte {
r := redaction.New(secrets)
safe := func(value string) string { return escapeMarkdown(r.String(value)) }
func Markdown(snapshot Snapshot) []byte {
var b strings.Builder
fmt.Fprintf(&b, "# Audita report: %s\n\n", safe(snapshot.RunID))
fmt.Fprintf(&b, "Status: %s\n\n", safe(snapshot.Status))
fmt.Fprintf(&b, "# Audita report: %s\n\n", escapeMarkdown(snapshot.RunID))
fmt.Fprintf(&b, "Status: %s\n\n", escapeMarkdown(snapshot.Status))
fmt.Fprintf(&b, "Corrections applied: %d \n", count(snapshot.Summary.CorrectionsApplied))
fmt.Fprintf(&b, "Work units: %s total; %s completed; %s interrupted; %s not started \n",
showCount(snapshot.Summary.WorkUnitsTotal), showCount(snapshot.Summary.WorkUnitsCompleted),
@@ -38,20 +34,20 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintf(&b, "Recoveries: %s of %s model operations \n", showCount(snapshot.Summary.Recoveries), showCount(snapshot.Summary.LLMOperations))
fmt.Fprintf(&b, "Proposal responses: %s; with filtering: %s\n\n",
showCount(snapshot.Summary.ProposalResponses), showCount(snapshot.Summary.ProposalResponsesWithFiltering))
fmt.Fprintf(&b, "Transcript input: %s \n", safe(snapshot.TranscriptPath))
fmt.Fprintf(&b, "Transcript input: %s \n", escapeMarkdown(snapshot.TranscriptPath))
output := snapshot.OutputPath
if output == "" {
output = "stdout"
}
fmt.Fprintf(&b, "Transcript output: %s\n\n", safe(output))
fmt.Fprintf(&b, "Transcript output: %s\n\n", escapeMarkdown(output))
fmt.Fprintln(&b, "## Work-unit errors")
fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, false)
writeMatrix(&b, snapshot, false)
fmt.Fprintln(&b)
fmt.Fprintln(&b, "## Recoveries")
fmt.Fprintln(&b)
writeMatrix(&b, snapshot, r, true)
writeMatrix(&b, snapshot, true)
fmt.Fprintln(&b)
if len(snapshot.RepairOutcomes) > 0 {
fmt.Fprintln(&b, "## Correction repair")
@@ -59,14 +55,14 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |")
fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |")
for _, row := range snapshot.RepairOutcomes {
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome))
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, escapeMarkdown(row.Outcome))
}
fmt.Fprintln(&b)
}
fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.")
fmt.Fprintln(&b)
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe)
writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries), safe)
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors))
writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries))
fmt.Fprintln(&b, "A recovery can have multiple reasons; reason counts need not sum to the recovery total.")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Details: `errors.yml` and `recoveries.yml`.")
@@ -80,7 +76,7 @@ func showCount(value *int) string {
return fmt.Sprint(*value)
}
func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redactor, recovery bool) {
func writeMatrix(b *strings.Builder, snapshot Snapshot, recovery bool) {
if len(snapshot.StageNames) == 0 || len(snapshot.WorkUnits) == 0 {
fmt.Fprintln(b, "No work units.")
return
@@ -88,7 +84,7 @@ func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redac
columns := snapshot.StageNames
fmt.Fprint(b, "| Chunk |")
for _, name := range columns {
fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(redactor.String(name))))
fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(name)))
}
fmt.Fprint(b, "\n| --- |")
for range columns {
@@ -137,7 +133,7 @@ var stageCellEscaper = strings.NewReplacer(
"://", "&#58;//", "www.", "www&#46;", "@", "&#64;",
)
func writeReasons(b *strings.Builder, title string, counts map[string]int, safe func(string) string) {
func writeReasons(b *strings.Builder, title string, counts map[string]int) {
fmt.Fprintf(b, "### %s\n\n", title)
if len(counts) == 0 {
fmt.Fprintln(b, "None.")
@@ -150,7 +146,7 @@ func writeReasons(b *strings.Builder, title string, counts map[string]int, safe
}
sort.Strings(keys)
for _, key := range keys {
fmt.Fprintf(b, "- `%s`: %d\n", safe(key), counts[key])
fmt.Fprintf(b, "- `%s`: %d\n", escapeMarkdown(key), counts[key])
}
fmt.Fprintln(b)
}

View File

@@ -28,7 +28,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
Errors: []ErrorRecord{{Scope: "work_unit", Failures: []Failure{{Reason: "transport_failed"}, {Reason: "transport_failed"}}}},
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
}
got := Markdown(snapshot, "secret")
got := Markdown(snapshot)
text := string(got)
// Compare only complete table rows, preserving order and states while leaving
// prose and spacing outside the fixed matrices free to change.
@@ -45,7 +45,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
}
for _, fragment := range []string{"in\\|put\\`[REDACTED]\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
for _, fragment := range []string{"in\\|put\\`secret\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
if !strings.Contains(text, fragment) {
t.Fatalf("missing %q in:\n%s", fragment, text)
}
@@ -69,9 +69,6 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
t.Fatalf("%s summary: got %v; want %v", label, got, want)
}
}
if strings.Contains(text, "secret") {
t.Fatal("secret leaked")
}
var decoded struct {
Summary Summary `yaml:"summary"`
}
@@ -88,12 +85,12 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
}
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}))
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
t.Fatal(text)
}
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
if !strings.Contains(text, "bad\\|label\\`<br>\\[REDACTED\\]") {
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}))
if !strings.Contains(text, "bad\\|label\\`<br>secret") {
t.Fatal(text)
}
}

View File

@@ -5,7 +5,6 @@ import (
"fmt"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gopkg.in/yaml.v3"
)
@@ -178,17 +177,13 @@ func completionTime(t *time.Time) *string {
return &value
}
// YAML redacts only string nodes after typed serialization.
func YAML(document any, secrets ...string) ([]byte, error) {
// YAML encodes the report document.
func YAML(document any) ([]byte, error) {
raw, err := yaml.Marshal(document)
if err != nil {
return nil, fmt.Errorf("encode report YAML: %w", err)
}
safe, err := redaction.New(secrets).YAML(raw)
if err != nil {
return nil, fmt.Errorf("redact report YAML: %w", err)
}
return safe, nil
return raw, nil
}
// StageLabel identifies a configured entry by position, module, and optional ID.

View File

@@ -8,7 +8,7 @@ import (
"gopkg.in/yaml.v3"
)
func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
func TestYAMLPreservesScalarTypes(t *testing.T) {
zero, one := 0, 1
falseValue := false
snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC),
@@ -16,18 +16,15 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}},
Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}}
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
payload, err := YAML(document, `secret\"key`, "1")
payload, err := YAML(document)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(payload), `secret\"key`) {
t.Fatalf("secret leaked: %s", payload)
}
var decoded map[string]any
if err := yaml.Unmarshal(payload, &decoded); err != nil {
t.Fatal(err)
}
if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
if decoded["run_id"] != snapshot.RunID || decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
t.Fatalf("wrong envelope types: %#v", decoded)
}
summary := decoded["summary"].(map[string]any)
@@ -35,7 +32,7 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
t.Fatalf("changed count types: %#v", summary)
}
}
if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
if payload, err := YAML(snapshot.RecoveriesDocument()); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
t.Fatalf("boolean changed: %s %v", payload, err)
}
}

View File

@@ -25,13 +25,11 @@ type CallHooks struct {
}
type RoleClient struct {
runtime *Runtime
role string
profileID string
settings config.LLM
hooks CallHooks
credentialMu sync.Mutex
credentialNames map[string]struct{}
runtime *Runtime
role string
profileID string
settings config.LLM
hooks CallHooks
}
var _ structuredoutput.Client = (*RoleClient)(nil)
@@ -152,14 +150,6 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var details pk.PreparedRun
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
details = prepared
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
c.credentialNames = make(map[string]struct{})
}
c.credentialNames[name] = struct{}{}
c.credentialMu.Unlock()
}
response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model
@@ -311,17 +301,6 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}
// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials.
func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
c.credentialMu.Lock()
names := make([]string, 0, len(c.credentialNames))
for name := range c.credentialNames {
names = append(names, name)
}
c.credentialMu.Unlock()
return c.runtime.ConfiguredSecrets(ctx, names...)
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages))
for i, message := range messages {

View File

@@ -1,13 +1,12 @@
package llm
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
@@ -23,13 +22,12 @@ type InteractionArtifacts struct {
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
type DiagnosticsWriter struct {
dir string
redactor redaction.Redactor
dir string
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,12 +80,12 @@ func sanitizeStageName(stage string) string {
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
redacted, err := w.redactor.JSON(payload)
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
}
redacted = append(redacted, '\n')
if err := os.WriteFile(path, redacted, 0o644); err != nil {
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
}
return nil

View File

@@ -4,13 +4,12 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir, nil)
writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction(
"validation:batch-1",
@@ -31,45 +30,6 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
}
func TestDiagnosticsWriterRedactsSecrets(t *testing.T) {
dir := t.TempDir()
secret := "top-secret-token"
writer := NewDiagnosticsWriter(dir, []string{secret})
artifacts, err := writer.WriteInteraction(
"module glossary / request",
map[string]any{"authorization": "Bearer " + secret},
map[string]any{"api_key": secret, "nested": map[string]any{"value": secret}},
map[string]any{"echo": secret},
map[string]any{"error": "failed with " + secret},
)
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
for _, path := range []string{
artifacts.RequestMetadataPath,
artifacts.RequestPayloadPath,
artifacts.ResponsePayloadPath,
artifacts.ErrorPayloadPath,
} {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read %s: %v", path, readErr)
}
content := string(raw)
if strings.Contains(content, secret) {
t.Fatalf("secret leaked in %s: %s", filepath.Base(path), content)
}
if !strings.Contains(content, "[REDACTED]") {
t.Fatalf("redaction marker missing in %s: %s", filepath.Base(path), content)
}
if !json.Valid(raw) {
t.Fatalf("invalid json in %s", filepath.Base(path))
}
}
}
func assertValidJSONFile(t *testing.T, path string) {
t.Helper()
raw, err := os.ReadFile(path)

View File

@@ -9,7 +9,6 @@ import (
"net/http"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -26,7 +25,7 @@ type transportFunc func(*http.Request) (*http.Response, error)
func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
func TestRoleClientUsesSelectedNativeCredential(t *testing.T) {
t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ")
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
@@ -41,7 +40,7 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
t.Fatal(err)
}
calls := 0
providerErr := errors.New("provider echoed synthetic-secret-value")
providerErr := errors.New("transport stopped")
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
calls++
if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" {
@@ -62,10 +61,6 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
if err == nil || calls != 1 {
t.Fatalf("error = %v, calls = %d", err, calls)
}
masked := runtime.RedactError(context.Background(), providerErr)
if strings.Contains(masked, "synthetic-secret-value") {
t.Fatal("credential not redacted")
}
}
func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
@@ -282,10 +277,4 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
}
})
}
secrets := runtime.ConfiguredSecrets(context.Background())
for _, value := range []string{"backend-secret-value", "profile-secret-value"} {
if !slices.Contains(secrets, value) {
t.Fatalf("missing credential masking value")
}
}
}

View File

@@ -8,7 +8,6 @@ import (
"net/http"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,12 +19,10 @@ import (
// Runtime owns the captured engine shared by all processing clients.
type Runtime struct {
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
credentialSourcesOnce sync.Once
credentialEnvNames map[string]struct{}
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
}
type AssetError struct{ Err error }

View File

@@ -1,78 +0,0 @@
package llm
import (
"context"
"os"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
// ConfiguredSecrets obtains only values referenced by captured credential metadata.
// It never selects a credential or treats an absent value as an error.
func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string {
if r == nil || r.Engine == nil {
return nil
}
r.credentialSourcesOnce.Do(func() {
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
})
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
read := func(name string) {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
for name := range r.credentialEnvNames {
read(name)
}
if len(additionalEnvNames) > 0 {
seen := make(map[string]struct{}, len(additionalEnvNames))
for _, name := range additionalEnvNames {
if _, known := r.credentialEnvNames[name]; known {
continue
}
if _, duplicate := seen[name]; name == "" || duplicate {
continue
}
seen[name] = struct{}{}
read(name)
}
}
return secrets
}
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
names := make(map[string]struct{})
add := func(name *string) {
if name != nil && *name != "" {
names[*name] = struct{}{}
}
}
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
for _, id := range ids {
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
add(profile.APIKeyEnv)
}
}
}
if ids, err := r.Engine.ListBackends(ctx); err == nil {
for _, id := range ids {
if backend, err := r.Engine.InspectBackend(ctx, id); err == nil {
add(backend.APIKeyEnv)
}
}
}
return names
}
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {
if err == nil {
return ""
}
return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error())
}

View File

@@ -1,52 +0,0 @@
package llm
import (
"context"
"os"
"path/filepath"
"slices"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
t.Setenv("SELECTED_KEY", "rotated-secret")
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
slices.Sort(values)
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
}
}

View File

@@ -28,7 +28,6 @@ type DebugInput struct {
RunID string
Metadata diagnostics.InvocationMetadata
EffectiveConfig []byte
Secrets []string
}
type Prepared struct {
@@ -85,7 +84,7 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
}
result := &Prepared{Output: session}
if input.Debug != nil {
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig, input.Debug.Secrets...)
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig)
}
if err := ctx.Err(); err != nil {
return nil, cancelOutput(err)

View File

@@ -130,17 +130,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
}
}

View File

@@ -19,7 +19,6 @@ import (
)
type fakeStructuredClient struct {
secrets []string
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
@@ -50,8 +49,6 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
return response, nil
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
type captureDiagnosticsWriter struct {
err error
lastStage string
@@ -311,21 +308,16 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
}
}
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "proposal-primary-secret"
validationSecret := "proposal-validation-secret"
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
client.secrets = []string{primarySecret, validationSecret}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}
req.Variables = map[string]string{"SectionJSON": "section text"}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -340,11 +332,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
if !json.Valid(raw) {
t.Fatalf("invalid JSON artifact %q: %s", path, raw)
}
}

View File

@@ -7,8 +7,6 @@ import (
"path/filepath"
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
}
current := original
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
findings := repairFindings(original.ProposalIndex, rejected)
redactor := redaction.New(repairSecrets(ctx, input))
for i := range findings {
findings[i].Explanation = redactor.String(findings[i].Explanation)
}
return findings
}
findings := safeFindings(rejected)
findings := repairFindings(original.ProposalIndex, rejected)
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
if !ok {
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
@@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input))
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
if writeErr == nil {
record.Artifacts = artifacts
@@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return nil, nil
}
if actionable(original.ProposalIndex, validated.rejected) {
findings = safeFindings(validated.rejected)
findings = repairFindings(original.ProposalIndex, validated.rejected)
return buildRequest()
}
branch.Outcome = "unavailable"
@@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
return finish(nil)
}
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
var secrets []string
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
if source, ok := client.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
}
}
return secrets
}

View File

@@ -31,7 +31,6 @@ type repairClient struct {
outputs []string
calls []structuredoutput.Request
err error
secrets []string
}
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
}
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
@@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
}
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
func TestRepairArtifactsAreUnique(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
@@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for _, path := range []string{a, b} {
for i, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
if !json.Valid(data) {
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
}
var request proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
t.Fatal(err)
}
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
t.Fatalf("repair request for proposal %d: %+v", i, request)
}
}
}
@@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
}
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
}
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "validation-secret") {
t.Fatalf("credential in artifact: %s", data)
}
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)

View File

@@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
if source, ok := input.ValidationLLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
}
}
diagnosticsWriter = interactionWriter
}
@@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
type llmDiagnosticsWriterAdapter struct {
dir string
secrets func() []string
dir string
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
var secrets []string
if a.secrets != nil {
secrets = a.secrets()
}
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
preparedSecret string
err error
calls int
responses []validators.LLMValidationResponse
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
f.calls++
if f.preparedSecret != "" {
f.secrets = append(f.secrets, f.preparedSecret)
f.preparedSecret = ""
}
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
@@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}
}
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
@@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
if !json.Valid(raw) {
t.Fatalf("invalid diagnostic %q: %s", path, raw)
}
}
@@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read decision diagnostic: %v", readErr)
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in diagnostics")
if !strings.Contains(string(raw), "supported by context") {
t.Fatalf("missing decision evidence: %s", raw)
}
}