Remove application credential substitution
This commit is contained in:
@@ -36,7 +36,7 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
}
|
||||
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||
if err != nil {
|
||||
return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err)
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
var warningsMu sync.Mutex
|
||||
@@ -63,24 +63,24 @@ func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcom
|
||||
warn(debug.WriteNormalizationSummary(summary))
|
||||
}
|
||||
if err := inv.Context.Err(); err != nil {
|
||||
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err)
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, err)
|
||||
}
|
||||
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
|
||||
if runErr != nil {
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
|
||||
}
|
||||
encoder, err := outputschema.Resolve(cfg.Output.Schema)
|
||||
if err != nil {
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
|
||||
}
|
||||
if result.Transcript == nil {
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
|
||||
}
|
||||
payload, err := encoder.Encoder(result.Transcript)
|
||||
if err != nil {
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, nil, err)
|
||||
}
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, payload, nil)
|
||||
}
|
||||
|
||||
// DebugPath returns the retained artifact directory, if its preparation succeeded.
|
||||
|
||||
@@ -87,10 +87,6 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
var secrets []string
|
||||
if cfg.Reporting.Enabled || input.Debug.Value {
|
||||
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
|
||||
}
|
||||
var initial diagnostics.ReportBundle
|
||||
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
|
||||
if cfg.Reporting.Enabled {
|
||||
@@ -98,7 +94,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
for index, item := range cfg.Pipeline.Stages {
|
||||
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
|
||||
}
|
||||
initial, err = renderReportBundle(snapshot, secrets...)
|
||||
initial, err = renderReportBundle(snapshot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -109,7 +105,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
if primary != nil {
|
||||
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
|
||||
}
|
||||
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
|
||||
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures))
|
||||
return bundle
|
||||
}
|
||||
}
|
||||
@@ -120,7 +116,7 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
if err := loaded.Configuration.WriteEffective(&effective); err != nil {
|
||||
warnings = append(warnings, err)
|
||||
}
|
||||
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||
}
|
||||
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
|
||||
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
|
||||
@@ -147,14 +143,14 @@ func generateRunID() (string, error) {
|
||||
return "run-" + hex.EncodeToString(random[:]), nil
|
||||
}
|
||||
|
||||
func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) {
|
||||
errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...)
|
||||
func renderReportBundle(value reporting.Snapshot) (diagnostics.ReportBundle, error) {
|
||||
errorsYAML, err := reporting.YAML(value.ErrorsDocument())
|
||||
if err != nil {
|
||||
return diagnostics.ReportBundle{}, err
|
||||
}
|
||||
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...)
|
||||
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument())
|
||||
if err != nil {
|
||||
return diagnostics.ReportBundle{}, err
|
||||
}
|
||||
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil
|
||||
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value)}, nil
|
||||
}
|
||||
|
||||
@@ -147,7 +147,11 @@ pipeline:
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) {
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, raw); err != nil {
|
||||
t.Fatalf("decode %s: %v", artifact, err)
|
||||
}
|
||||
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
|
||||
t.Fatalf("%s missing attribution: %s", artifact, raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,19 +7,16 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
// finishProcess delivers the transcript and reports before emitting success information.
|
||||
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
|
||||
func finishProcess(inv commandkit.Invocation, input processInput, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
|
||||
reporter := inv.Reporter()
|
||||
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
|
||||
safe := redaction.New(secrets)
|
||||
for _, warning := range prepared.Prepared.Warnings {
|
||||
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
|
||||
_ = reporter.Warn("Debug artifact: " + warning.Error())
|
||||
}
|
||||
completed := time.Now().UTC()
|
||||
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
|
||||
@@ -36,7 +33,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
|
||||
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
|
||||
if len(reportPaths) > 0 {
|
||||
var err error
|
||||
reports, err = renderReportBundle(snapshot, secrets...)
|
||||
reports, err = renderReportBundle(snapshot)
|
||||
if err != nil {
|
||||
if primary == nil {
|
||||
primary = err
|
||||
@@ -44,7 +41,7 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
|
||||
snapshot.Status = reporting.StatusFailed
|
||||
}
|
||||
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
|
||||
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
|
||||
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures))
|
||||
if bundleErr != nil {
|
||||
return reports
|
||||
}
|
||||
@@ -60,14 +57,14 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
|
||||
}
|
||||
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
|
||||
if result != nil {
|
||||
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
|
||||
return commandkit.Failure(result.Error()).WithCause(result)
|
||||
}
|
||||
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
|
||||
for _, path := range reportPaths {
|
||||
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))
|
||||
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", path))
|
||||
}
|
||||
if prepared.Prepared.Debug != nil {
|
||||
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path())))
|
||||
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", prepared.Prepared.Debug.Path()))
|
||||
}
|
||||
return commandkit.Success()
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestPreparedDeliveryFollowsReportingMode(t *testing.T) {
|
||||
if err != nil {
|
||||
return commandkit.Failure(err.Error())
|
||||
}
|
||||
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||
}})
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||
if err != nil {
|
||||
@@ -97,7 +97,7 @@ func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) {
|
||||
if err != nil {
|
||||
return commandkit.Failure(err.Error())
|
||||
}
|
||||
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||
return finishProcess(inv, input, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
|
||||
}})
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||
if err != nil {
|
||||
@@ -161,7 +161,7 @@ func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel()
|
||||
return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil)
|
||||
return finishProcess(inv, input, prepared, result.Snapshot, []byte("new transcript"), nil)
|
||||
}})
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
@@ -25,7 +24,6 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
||||
}
|
||||
|
||||
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
||||
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
|
||||
definitions := map[string]string{
|
||||
"malformed operator YAML": "profiles: [\n",
|
||||
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
|
||||
@@ -38,9 +36,6 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
|
||||
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
|
||||
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
|
||||
}
|
||||
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
|
||||
t.Fatal("configured secret leaked")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func (f FileFailure) Error() string { return fmt.Sprintf("%s %q: %v", f.Kind, f.
|
||||
func (f FileFailure) Unwrap() error { return f.Err }
|
||||
|
||||
// OutputSession owns checked destinations and per-file publication. Filesystem
|
||||
// failures remain typed so CLI can present safe, redacted diagnostics.
|
||||
// failures remain typed so CLI can present contextual diagnostics.
|
||||
type OutputSession struct {
|
||||
transcript string
|
||||
reports [3]string
|
||||
@@ -250,8 +250,8 @@ func (s *OutputSession) Initialize(initial ReportBundle, failed func([]FileFailu
|
||||
}
|
||||
|
||||
// PrepareDebug creates optional initial diagnostics and returns any nonfatal problems.
|
||||
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte, secrets ...string) (*RunDirectory, []error) {
|
||||
run, err := NewRunDirectory(workDir, runID, secrets...)
|
||||
func (s *OutputSession) PrepareDebug(workDir, runID string, metadata InvocationMetadata, effective []byte) (*RunDirectory, []error) {
|
||||
run, err := NewRunDirectory(workDir, runID)
|
||||
if err != nil {
|
||||
return nil, []error{err}
|
||||
}
|
||||
|
||||
@@ -2,13 +2,17 @@ package diagnostics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestPrepareOutputAllowsInPlaceTranscriptButRejectsReportCollisions(t *testing.T) {
|
||||
@@ -242,7 +246,7 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
|
||||
if _, err := os.Stat(parent); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("unexpected parent: %v", err)
|
||||
}
|
||||
run, err := NewRunDirectory(parent, "run-123", "secret")
|
||||
run, err := NewRunDirectory(parent, "run-123")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -252,11 +256,11 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
|
||||
if _, err := NewRunDirectory(parent, "run-123"); err == nil {
|
||||
t.Fatal("duplicate run ID accepted")
|
||||
}
|
||||
if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "secret"}); err != nil {
|
||||
if err := run.WriteJSONArtifact(ArtifactCorrectionLedger, map[string]string{"proposal": "edit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents, err := os.ReadFile(filepath.Join(run.Path(), ArtifactCorrectionLedger))
|
||||
if err != nil || strings.Contains(string(contents), "secret") {
|
||||
if err != nil || string(contents) != "{\"proposal\":\"edit\"}\n" {
|
||||
t.Fatalf("debug artifact: %s %v", contents, err)
|
||||
}
|
||||
if _, err := os.Stat(run.Path()); err != nil {
|
||||
@@ -264,6 +268,48 @@ func TestDebugRunDirectoryUsesSuppliedIDAndIsRetained(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugTranscriptSnapshotsPreserveSourceAndTypedViews(t *testing.T) {
|
||||
run, err := NewRunDirectory(t.TempDir(), "run-transcripts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := []byte(" [\n {\"id\": 2, \"text\": \"literal\", \"speaker\": \"A\", \"start\": 0, \"end\": 1}\n ] ")
|
||||
source, err := schema.ParseSourceTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run.WriteSourceTranscript(source, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
normalized := &schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "normalized"}}}
|
||||
if err := run.WriteNormalizedTranscript(normalized); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
acquired, err := os.ReadFile(filepath.Join(run.Path(), ArtifactSourceTranscript))
|
||||
if err != nil || !bytes.Equal(acquired, raw) {
|
||||
t.Fatalf("acquired source = %q, error = %v", acquired, err)
|
||||
}
|
||||
for name, want := range map[string]string{ArtifactParsedSourceTranscript: "literal", ArtifactNormalizedTranscript: "normalized"} {
|
||||
payload, err := os.ReadFile(filepath.Join(run.Path(), name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var transcript schema.Transcript
|
||||
if err := json.Unmarshal(payload, &transcript); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(transcript.Segments) != 1 || transcript.Segments[0].ID != 2 || transcript.Segments[0].Text != want {
|
||||
t.Fatalf("%s: %s", name, payload)
|
||||
}
|
||||
}
|
||||
if err := run.WriteJSONArtifact("invalid.json", math.NaN()); err == nil {
|
||||
t.Fatal("expected serialization failure")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("failed artifact persisted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectionAndAbortPreserveTranscript(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
output := filepath.Join(dir, "output.json")
|
||||
@@ -363,7 +409,7 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
|
||||
if run, problems := session.PrepareDebug(file, "run", InvocationMetadata{}, nil); run != nil || len(problems) == 0 {
|
||||
t.Fatalf("run=%v problems=%v", run, problems)
|
||||
}
|
||||
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("secret: sensitive\n"), "sensitive")
|
||||
run, problems := session.PrepareDebug(filepath.Join(dir, "debug"), "run", InvocationMetadata{Operation: "process"}, []byte("version: 3\n"))
|
||||
if len(problems) != 0 {
|
||||
t.Fatal(problems)
|
||||
}
|
||||
@@ -371,8 +417,8 @@ func TestOptionalDebugPreparationReportsProblems(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(contents), "sensitive") {
|
||||
t.Fatal("debug artifact contains secret")
|
||||
if string(contents) != "version: 3\n" {
|
||||
t.Fatalf("changed effective configuration: %q", contents)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,13 +9,11 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
// RunDirectory represents a per-run diagnostics directory
|
||||
type RunDirectory struct {
|
||||
redactor redaction.Redactor
|
||||
path string
|
||||
createdAt time.Time
|
||||
}
|
||||
@@ -32,7 +30,7 @@ type InvocationMetadata struct {
|
||||
}
|
||||
|
||||
// NewRunDirectory creates a new run directory under the configured work dir
|
||||
func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, error) {
|
||||
func NewRunDirectory(workDir, runID string) (*RunDirectory, error) {
|
||||
if strings.TrimSpace(workDir) == "" {
|
||||
return nil, fmt.Errorf("debug work directory is required")
|
||||
}
|
||||
@@ -53,7 +51,6 @@ func NewRunDirectory(workDir, runID string, secrets ...string) (*RunDirectory, e
|
||||
|
||||
return &RunDirectory{
|
||||
path: runPath,
|
||||
redactor: redaction.New(secrets),
|
||||
createdAt: time.Now().UTC(),
|
||||
}, nil
|
||||
}
|
||||
@@ -77,7 +74,7 @@ func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) erro
|
||||
}
|
||||
|
||||
path := filepath.Join(r.path, ArtifactInvocationMetadata)
|
||||
bytes, err := r.redactor.JSON(metadata)
|
||||
bytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal invocation metadata: %w", err)
|
||||
}
|
||||
@@ -95,25 +92,14 @@ func (r *RunDirectory) WriteEffectiveConfig(effectiveYAML []byte) error {
|
||||
|
||||
// WriteSourceTranscript writes the source transcript artifact
|
||||
func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript, raw []byte) error {
|
||||
// Preserve the source snapshot without copying unredacted input bytes.
|
||||
var source any
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&source); err != nil {
|
||||
return fmt.Errorf("decode source snapshot: %w", err)
|
||||
}
|
||||
sourceBytes, err := r.redactor.JSON(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("redact source snapshot: %w", err)
|
||||
}
|
||||
sourcePath := filepath.Join(r.path, ArtifactSourceTranscript)
|
||||
if err := os.WriteFile(sourcePath, append(sourceBytes, '\n'), 0o644); err != nil {
|
||||
if err := os.WriteFile(sourcePath, raw, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write source transcript: %w", err)
|
||||
}
|
||||
|
||||
// Write parsed source for debugging
|
||||
parsedPath := filepath.Join(r.path, ArtifactParsedSourceTranscript)
|
||||
parsedBytes, err := r.redactor.JSON(transcript)
|
||||
parsedBytes, err := json.Marshal(transcript)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal parsed source transcript: %w", err)
|
||||
}
|
||||
@@ -128,7 +114,7 @@ func (r *RunDirectory) WriteSourceTranscript(transcript *schema.SourceTranscript
|
||||
// WriteNormalizedTranscript writes the normalized transcript artifact
|
||||
func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript) error {
|
||||
normalizedPath := filepath.Join(r.path, ArtifactNormalizedTranscript)
|
||||
bytes, err := r.redactor.JSON(transcript)
|
||||
bytes, err := json.Marshal(transcript)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to serialize normalized transcript: %w", err)
|
||||
}
|
||||
@@ -141,7 +127,7 @@ func (r *RunDirectory) WriteNormalizedTranscript(transcript *schema.Transcript)
|
||||
// WriteNormalizationSummary writes the normalization summary artifact
|
||||
func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.NormalizationSummary) error {
|
||||
summaryPath := filepath.Join(r.path, ArtifactNormalizationSummary)
|
||||
bytes, err := r.redactor.JSON(summary)
|
||||
bytes, err := json.Marshal(summary)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal normalization summary: %w", err)
|
||||
}
|
||||
@@ -154,7 +140,7 @@ func (r *RunDirectory) WriteNormalizationSummary(summary *normalization.Normaliz
|
||||
|
||||
func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
||||
artifactPath := filepath.Join(r.path, name)
|
||||
bytes, err := r.redactor.JSON(payload)
|
||||
bytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal %s: %w", name, err)
|
||||
}
|
||||
@@ -165,13 +151,8 @@ func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteYAMLArtifact writes a sanitized YAML diagnostic artifact.
|
||||
// WriteYAMLArtifact writes the supplied YAML diagnostic artifact.
|
||||
func (r *RunDirectory) WriteYAMLArtifact(name string, payload []byte) error {
|
||||
var err error
|
||||
payload, err = r.redactor.YAML(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(r.path, name), payload, 0o644); err != nil {
|
||||
return fmt.Errorf("failed to write %s: %w", name, err)
|
||||
}
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
// Package redaction sanitizes diagnostic representations without modifying domain data.
|
||||
package redaction
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Redactor replaces configured values in a single pass, longest matches first.
|
||||
// Its zero value is usable and leaves strings unchanged.
|
||||
type Redactor struct{ replacer *strings.Replacer }
|
||||
|
||||
func New(secrets []string) Redactor {
|
||||
values := map[string]bool{}
|
||||
for _, secret := range secrets {
|
||||
if secret == "" {
|
||||
continue
|
||||
}
|
||||
values[secret] = true
|
||||
// Provider content may itself be serialized JSON inside a string field.
|
||||
encoded, _ := json.Marshal(secret)
|
||||
values[string(encoded[1:len(encoded)-1])] = true
|
||||
}
|
||||
keys := make([]string, 0, len(values))
|
||||
for value := range values {
|
||||
keys = append(keys, value)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) == len(keys[j]) {
|
||||
return keys[i] < keys[j]
|
||||
}
|
||||
return len(keys[i]) > len(keys[j])
|
||||
})
|
||||
pairs := make([]string, 0, 2*len(keys))
|
||||
for _, key := range keys {
|
||||
pairs = append(pairs, key, "[REDACTED]")
|
||||
}
|
||||
return Redactor{replacer: strings.NewReplacer(pairs...)}
|
||||
}
|
||||
|
||||
func (r Redactor) String(value string) string {
|
||||
if r.replacer == nil {
|
||||
return value
|
||||
}
|
||||
return r.replacer.Replace(value)
|
||||
}
|
||||
|
||||
// JSON serializes first, then redacts only decoded string values and keys.
|
||||
// Numbers retain their exact representation. Failure never returns original data.
|
||||
func (r Redactor) JSON(value any) ([]byte, error) {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize diagnostic: %s", r.String(err.Error()))
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber()
|
||||
var decoded any
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode diagnostic: %s", r.String(err.Error()))
|
||||
}
|
||||
safe, err := r.walk(decoded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.MarshalIndent(safe, "", " ")
|
||||
}
|
||||
|
||||
func (r Redactor) walk(value any) (any, error) {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return r.String(v), nil
|
||||
case []any:
|
||||
for i := range v {
|
||||
var err error
|
||||
v[i], err = r.walk(v[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
case map[string]any:
|
||||
safe := make(map[string]any, len(v))
|
||||
for key, item := range v {
|
||||
safeKey := r.String(key)
|
||||
if _, exists := safe[safeKey]; exists {
|
||||
return nil, fmt.Errorf("redaction would produce duplicate object keys")
|
||||
}
|
||||
safeItem, err := r.walk(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
safe[safeKey] = safeItem
|
||||
}
|
||||
return safe, nil
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// YAML preserves scalar types while sanitizing string nodes, including map keys.
|
||||
func (r Redactor) YAML(raw []byte) ([]byte, error) {
|
||||
var node yaml.Node
|
||||
if err := yaml.Unmarshal(raw, &node); err != nil {
|
||||
return nil, fmt.Errorf("decode diagnostic YAML: %s", r.String(err.Error()))
|
||||
}
|
||||
var visit func(*yaml.Node)
|
||||
visit = func(n *yaml.Node) {
|
||||
if n.Kind == yaml.ScalarNode && n.Tag == "!!str" {
|
||||
n.Value = r.String(n.Value)
|
||||
}
|
||||
n.HeadComment = r.String(n.HeadComment)
|
||||
n.LineComment = r.String(n.LineComment)
|
||||
n.FootComment = r.String(n.FootComment)
|
||||
for _, child := range n.Content {
|
||||
visit(child)
|
||||
}
|
||||
}
|
||||
visit(&node)
|
||||
return yaml.Marshal(&node)
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package redaction_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestStringRedactionPreservesSerializationAndDomainData(t *testing.T) {
|
||||
secrets := []string{"2", `quoted"\key`, "overlap", "overlap-long", "REDACTED"}
|
||||
r := redaction.New(secrets)
|
||||
original := map[string]any{"number": 2, "large": json.Number("9007199254740993"), "bool": true, "text": `echo:2 quoted"\key overlap-long overlap`}
|
||||
raw, err := r.JSON(original)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded["number"] != float64(2) || decoded["bool"] != true || !strings.Contains(string(raw), "9007199254740993") {
|
||||
t.Fatalf("changed scalar meanings: %s", raw)
|
||||
}
|
||||
if decoded["text"] != "echo:[REDACTED] [REDACTED] [REDACTED] [REDACTED]" {
|
||||
t.Fatalf("redaction: %s", raw)
|
||||
}
|
||||
if original["text"] != `echo:2 quoted"\key overlap-long overlap` {
|
||||
t.Fatal("mutated domain data")
|
||||
}
|
||||
if raw, err := r.JSON(math.NaN()); err == nil || raw != nil {
|
||||
t.Fatalf("unsafe fallback: %s %v", raw, err)
|
||||
}
|
||||
if raw, err := r.JSON(map[string]string{"overlap": "first", "overlap-long": "second"}); err == nil || raw != nil {
|
||||
t.Fatalf("colliding redacted keys: %s %v", raw, err)
|
||||
}
|
||||
nested, _ := json.Marshal(map[string]string{"reason": `quoted"\key`})
|
||||
raw, err = r.JSON(map[string]string{"final_content": string(nested)})
|
||||
if err != nil || strings.Contains(string(raw), "key") {
|
||||
t.Fatalf("nested content: %s %v", raw, err)
|
||||
}
|
||||
yaml, err := r.YAML([]byte("version: 2\nenabled: true\nmessage: echo:2\n"))
|
||||
if err != nil || !strings.Contains(string(yaml), "version: 2") || !strings.Contains(string(yaml), "echo:[REDACTED]") {
|
||||
t.Fatalf("YAML: %s %v", yaml, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebugWritersRedactSourceAndNormalizedSnapshots(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
run, err := diagnostics.NewRunDirectory(dir, "run-test", "2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run.WriteSourceTranscript(&schema.SourceTranscript{}, []byte(`{"segments":[{"text":"echo:2","id":2}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run.WriteNormalizedTranscript(&schema.Transcript{Segments: []schema.Segment{{ID: 2, Text: "echo:2"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, path := range []string{filepath.Join(run.Path(), diagnostics.ArtifactSourceTranscript), filepath.Join(run.Path(), diagnostics.ArtifactNormalizedTranscript)} {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), "echo:2") || !strings.Contains(string(raw), "echo:[REDACTED]") || !strings.Contains(string(raw), `"id": 2`) {
|
||||
t.Fatalf("snapshot: %s", raw)
|
||||
}
|
||||
}
|
||||
if err := run.WriteJSONArtifact("invalid.json", map[string]any{"secret": "echo:2", "invalid": math.NaN()}); err == nil {
|
||||
t.Fatal("expected serialization failure")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(run.Path(), "invalid.json")); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed artifact persisted: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"html"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
)
|
||||
|
||||
// CompletionLine is the brief successful CLI summary. It uses the same counts
|
||||
@@ -24,12 +22,10 @@ func count(value *int) int {
|
||||
}
|
||||
|
||||
// Markdown renders a fixed-layout human report from one processing snapshot.
|
||||
func Markdown(snapshot Snapshot, secrets ...string) []byte {
|
||||
r := redaction.New(secrets)
|
||||
safe := func(value string) string { return escapeMarkdown(r.String(value)) }
|
||||
func Markdown(snapshot Snapshot) []byte {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "# Audita report: %s\n\n", safe(snapshot.RunID))
|
||||
fmt.Fprintf(&b, "Status: %s\n\n", safe(snapshot.Status))
|
||||
fmt.Fprintf(&b, "# Audita report: %s\n\n", escapeMarkdown(snapshot.RunID))
|
||||
fmt.Fprintf(&b, "Status: %s\n\n", escapeMarkdown(snapshot.Status))
|
||||
fmt.Fprintf(&b, "Corrections applied: %d \n", count(snapshot.Summary.CorrectionsApplied))
|
||||
fmt.Fprintf(&b, "Work units: %s total; %s completed; %s interrupted; %s not started \n",
|
||||
showCount(snapshot.Summary.WorkUnitsTotal), showCount(snapshot.Summary.WorkUnitsCompleted),
|
||||
@@ -38,20 +34,20 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
|
||||
fmt.Fprintf(&b, "Recoveries: %s of %s model operations \n", showCount(snapshot.Summary.Recoveries), showCount(snapshot.Summary.LLMOperations))
|
||||
fmt.Fprintf(&b, "Proposal responses: %s; with filtering: %s\n\n",
|
||||
showCount(snapshot.Summary.ProposalResponses), showCount(snapshot.Summary.ProposalResponsesWithFiltering))
|
||||
fmt.Fprintf(&b, "Transcript input: %s \n", safe(snapshot.TranscriptPath))
|
||||
fmt.Fprintf(&b, "Transcript input: %s \n", escapeMarkdown(snapshot.TranscriptPath))
|
||||
output := snapshot.OutputPath
|
||||
if output == "" {
|
||||
output = "stdout"
|
||||
}
|
||||
fmt.Fprintf(&b, "Transcript output: %s\n\n", safe(output))
|
||||
fmt.Fprintf(&b, "Transcript output: %s\n\n", escapeMarkdown(output))
|
||||
|
||||
fmt.Fprintln(&b, "## Work-unit errors")
|
||||
fmt.Fprintln(&b)
|
||||
writeMatrix(&b, snapshot, r, false)
|
||||
writeMatrix(&b, snapshot, false)
|
||||
fmt.Fprintln(&b)
|
||||
fmt.Fprintln(&b, "## Recoveries")
|
||||
fmt.Fprintln(&b)
|
||||
writeMatrix(&b, snapshot, r, true)
|
||||
writeMatrix(&b, snapshot, true)
|
||||
fmt.Fprintln(&b)
|
||||
if len(snapshot.RepairOutcomes) > 0 {
|
||||
fmt.Fprintln(&b, "## Correction repair")
|
||||
@@ -59,14 +55,14 @@ func Markdown(snapshot Snapshot, secrets ...string) []byte {
|
||||
fmt.Fprintln(&b, "| Stage | Chunk | Proposal index | Attempts | Outcome |")
|
||||
fmt.Fprintln(&b, "| ---: | ---: | ---: | ---: | --- |")
|
||||
for _, row := range snapshot.RepairOutcomes {
|
||||
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, safe(row.Outcome))
|
||||
fmt.Fprintf(&b, "| %d | %d | %d | %d | %s |\n", row.StageIndex+1, row.ChunkIndex+1, row.ProposalIndex, row.Attempts, escapeMarkdown(row.Outcome))
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
}
|
||||
fmt.Fprintln(&b, "Legend: `—` = not started; `n*` = interrupted with n recorded errors or recoveries. Completed error cells are 0 or 1.")
|
||||
fmt.Fprintln(&b)
|
||||
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors), safe)
|
||||
writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries), safe)
|
||||
writeReasons(&b, "Error reasons", errorReasons(snapshot.Errors))
|
||||
writeReasons(&b, "Recovery reasons", recoveryReasons(snapshot.Recoveries))
|
||||
fmt.Fprintln(&b, "A recovery can have multiple reasons; reason counts need not sum to the recovery total.")
|
||||
fmt.Fprintln(&b)
|
||||
fmt.Fprintln(&b, "Details: `errors.yml` and `recoveries.yml`.")
|
||||
@@ -80,7 +76,7 @@ func showCount(value *int) string {
|
||||
return fmt.Sprint(*value)
|
||||
}
|
||||
|
||||
func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redactor, recovery bool) {
|
||||
func writeMatrix(b *strings.Builder, snapshot Snapshot, recovery bool) {
|
||||
if len(snapshot.StageNames) == 0 || len(snapshot.WorkUnits) == 0 {
|
||||
fmt.Fprintln(b, "No work units.")
|
||||
return
|
||||
@@ -88,7 +84,7 @@ func writeMatrix(b *strings.Builder, snapshot Snapshot, redactor redaction.Redac
|
||||
columns := snapshot.StageNames
|
||||
fmt.Fprint(b, "| Chunk |")
|
||||
for _, name := range columns {
|
||||
fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(redactor.String(name))))
|
||||
fmt.Fprintf(b, " %s |", stageCellEscaper.Replace(html.EscapeString(name)))
|
||||
}
|
||||
fmt.Fprint(b, "\n| --- |")
|
||||
for range columns {
|
||||
@@ -137,7 +133,7 @@ var stageCellEscaper = strings.NewReplacer(
|
||||
"://", "://", "www.", "www.", "@", "@",
|
||||
)
|
||||
|
||||
func writeReasons(b *strings.Builder, title string, counts map[string]int, safe func(string) string) {
|
||||
func writeReasons(b *strings.Builder, title string, counts map[string]int) {
|
||||
fmt.Fprintf(b, "### %s\n\n", title)
|
||||
if len(counts) == 0 {
|
||||
fmt.Fprintln(b, "None.")
|
||||
@@ -150,7 +146,7 @@ func writeReasons(b *strings.Builder, title string, counts map[string]int, safe
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
fmt.Fprintf(b, "- `%s`: %d\n", safe(key), counts[key])
|
||||
fmt.Fprintf(b, "- `%s`: %d\n", escapeMarkdown(key), counts[key])
|
||||
}
|
||||
fmt.Fprintln(b)
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
Errors: []ErrorRecord{{Scope: "work_unit", Failures: []Failure{{Reason: "transport_failed"}, {Reason: "transport_failed"}}}},
|
||||
Recoveries: []RecoveryRecord{{Reasons: []string{"output_repaired", "proposals_filtered"}}, {Reasons: []string{"output_repaired"}}, {Reasons: []string{"service_recovered"}}},
|
||||
}
|
||||
got := Markdown(snapshot, "secret")
|
||||
got := Markdown(snapshot)
|
||||
text := string(got)
|
||||
// Compare only complete table rows, preserving order and states while leaving
|
||||
// prose and spacing outside the fixed matrices free to change.
|
||||
@@ -45,7 +45,7 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
if rows := markdownTableRows(text); !reflect.DeepEqual(rows, wantRows) {
|
||||
t.Fatalf("matrices: got %v; want %v", rows, wantRows)
|
||||
}
|
||||
for _, fragment := range []string{"in\\|put\\`[REDACTED]\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
|
||||
for _, fragment := range []string{"in\\|put\\`secret\\`.json", "out<br>file.json", "`transport_failed`: 1", "`output_repaired`: 2", "`proposals_filtered`: 1", "`service_recovered`: 1"} {
|
||||
if !strings.Contains(text, fragment) {
|
||||
t.Fatalf("missing %q in:\n%s", fragment, text)
|
||||
}
|
||||
@@ -69,9 +69,6 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
t.Fatalf("%s summary: got %v; want %v", label, got, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, "secret") {
|
||||
t.Fatal("secret leaked")
|
||||
}
|
||||
var decoded struct {
|
||||
Summary Summary `yaml:"summary"`
|
||||
}
|
||||
@@ -88,12 +85,12 @@ func TestMarkdownFixedLayoutAndCounts(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMarkdownNoWorkAndEscapedStageLabel(t *testing.T) {
|
||||
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}, "secret"))
|
||||
text := string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}}))
|
||||
if len(markdownTableRows(text)) != 0 || strings.Contains(text, "secret") {
|
||||
t.Fatal(text)
|
||||
}
|
||||
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}, "secret"))
|
||||
if !strings.Contains(text, "bad\\|label\\`<br>\\[REDACTED\\]") {
|
||||
text = string(Markdown(Snapshot{RunID: "r", StageNames: []string{"bad|label`\nsecret"}, WorkUnits: []WorkUnit{{StageIndex: 0, ChunkIndex: 0, State: "completed"}}}))
|
||||
if !strings.Contains(text, "bad\\|label\\`<br>secret") {
|
||||
t.Fatal(text)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -178,17 +177,13 @@ func completionTime(t *time.Time) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
// YAML redacts only string nodes after typed serialization.
|
||||
func YAML(document any, secrets ...string) ([]byte, error) {
|
||||
// YAML encodes the report document.
|
||||
func YAML(document any) ([]byte, error) {
|
||||
raw, err := yaml.Marshal(document)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode report YAML: %w", err)
|
||||
}
|
||||
safe, err := redaction.New(secrets).YAML(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("redact report YAML: %w", err)
|
||||
}
|
||||
return safe, nil
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// StageLabel identifies a configured entry by position, module, and optional ID.
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
|
||||
func TestYAMLPreservesScalarTypes(t *testing.T) {
|
||||
zero, one := 0, 1
|
||||
falseValue := false
|
||||
snapshot := Snapshot{RunID: "secret\\\"key", Status: StatusCompleted, StartedAt: time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC),
|
||||
@@ -16,18 +16,15 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
|
||||
Errors: []ErrorRecord{{Scope: "run", Phase: "processing", Reason: "internal_failure", Message: "secret\\\"key 1"}},
|
||||
Recoveries: []RecoveryRecord{{StageIndex: 0, Reasons: []string{"service_recovered"}, ExecutionCounts: ExecutionCounts{RetriesExhausted: &falseValue}}}}
|
||||
for _, document := range []any{snapshot.ErrorsDocument(), snapshot.RecoveriesDocument()} {
|
||||
payload, err := YAML(document, `secret\"key`, "1")
|
||||
payload, err := YAML(document)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(payload), `secret\"key`) {
|
||||
t.Fatalf("secret leaked: %s", payload)
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := yaml.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
|
||||
if decoded["run_id"] != snapshot.RunID || decoded["version"] != 1 || decoded["started_at"] != "2026-09-24T00:00:00Z" {
|
||||
t.Fatalf("wrong envelope types: %#v", decoded)
|
||||
}
|
||||
summary := decoded["summary"].(map[string]any)
|
||||
@@ -35,7 +32,7 @@ func TestYAMLRedactsStringsWithoutChangingScalarTypes(t *testing.T) {
|
||||
t.Fatalf("changed count types: %#v", summary)
|
||||
}
|
||||
}
|
||||
if payload, err := YAML(snapshot.RecoveriesDocument(), "1"); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
|
||||
if payload, err := YAML(snapshot.RecoveriesDocument()); err != nil || !strings.Contains(string(payload), "retries_exhausted: false") {
|
||||
t.Fatalf("boolean changed: %s %v", payload, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,13 +25,11 @@ type CallHooks struct {
|
||||
}
|
||||
|
||||
type RoleClient struct {
|
||||
runtime *Runtime
|
||||
role string
|
||||
profileID string
|
||||
settings config.LLM
|
||||
hooks CallHooks
|
||||
credentialMu sync.Mutex
|
||||
credentialNames map[string]struct{}
|
||||
runtime *Runtime
|
||||
role string
|
||||
profileID string
|
||||
settings config.LLM
|
||||
hooks CallHooks
|
||||
}
|
||||
|
||||
var _ structuredoutput.Client = (*RoleClient)(nil)
|
||||
@@ -152,14 +150,6 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
var details pk.PreparedRun
|
||||
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
|
||||
details = prepared
|
||||
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
|
||||
c.credentialMu.Lock()
|
||||
if c.credentialNames == nil {
|
||||
c.credentialNames = make(map[string]struct{})
|
||||
}
|
||||
c.credentialNames[name] = struct{}{}
|
||||
c.credentialMu.Unlock()
|
||||
}
|
||||
response.Prompt = promptMetadata(details)
|
||||
record.Prompt = response.Prompt
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
@@ -311,17 +301,6 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot
|
||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
||||
}
|
||||
|
||||
// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials.
|
||||
func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
|
||||
c.credentialMu.Lock()
|
||||
names := make([]string, 0, len(c.credentialNames))
|
||||
for name := range c.credentialNames {
|
||||
names = append(names, name)
|
||||
}
|
||||
c.credentialMu.Unlock()
|
||||
return c.runtime.ConfiguredSecrets(ctx, names...)
|
||||
}
|
||||
|
||||
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
|
||||
result := make([]structuredoutput.Message, len(messages))
|
||||
for i, message := range messages {
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
)
|
||||
|
||||
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
|
||||
@@ -23,13 +22,12 @@ type InteractionArtifacts struct {
|
||||
|
||||
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
|
||||
type DiagnosticsWriter struct {
|
||||
dir string
|
||||
redactor redaction.Redactor
|
||||
dir string
|
||||
}
|
||||
|
||||
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
|
||||
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
|
||||
return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
|
||||
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
|
||||
return &DiagnosticsWriter{dir: dir}
|
||||
}
|
||||
|
||||
// WriteInteraction writes request/response/error artifacts for a generic LLM
|
||||
@@ -82,12 +80,12 @@ func sanitizeStageName(stage string) string {
|
||||
}
|
||||
|
||||
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
|
||||
redacted, err := w.redactor.JSON(payload)
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
|
||||
}
|
||||
redacted = append(redacted, '\n')
|
||||
if err := os.WriteFile(path, redacted, 0o644); err != nil {
|
||||
encoded = append(encoded, '\n')
|
||||
if err := os.WriteFile(path, encoded, 0o644); err != nil {
|
||||
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -4,13 +4,12 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
writer := NewDiagnosticsWriter(dir, nil)
|
||||
writer := NewDiagnosticsWriter(dir)
|
||||
|
||||
artifacts, err := writer.WriteInteraction(
|
||||
"validation:batch-1",
|
||||
@@ -31,45 +30,6 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
|
||||
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
|
||||
}
|
||||
|
||||
func TestDiagnosticsWriterRedactsSecrets(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
secret := "top-secret-token"
|
||||
writer := NewDiagnosticsWriter(dir, []string{secret})
|
||||
|
||||
artifacts, err := writer.WriteInteraction(
|
||||
"module glossary / request",
|
||||
map[string]any{"authorization": "Bearer " + secret},
|
||||
map[string]any{"api_key": secret, "nested": map[string]any{"value": secret}},
|
||||
map[string]any{"echo": secret},
|
||||
map[string]any{"error": "failed with " + secret},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteInteraction: %v", err)
|
||||
}
|
||||
|
||||
for _, path := range []string{
|
||||
artifacts.RequestMetadataPath,
|
||||
artifacts.RequestPayloadPath,
|
||||
artifacts.ResponsePayloadPath,
|
||||
artifacts.ErrorPayloadPath,
|
||||
} {
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read %s: %v", path, readErr)
|
||||
}
|
||||
content := string(raw)
|
||||
if strings.Contains(content, secret) {
|
||||
t.Fatalf("secret leaked in %s: %s", filepath.Base(path), content)
|
||||
}
|
||||
if !strings.Contains(content, "[REDACTED]") {
|
||||
t.Fatalf("redaction marker missing in %s: %s", filepath.Base(path), content)
|
||||
}
|
||||
if !json.Valid(raw) {
|
||||
t.Fatalf("invalid json in %s", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertValidJSONFile(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -26,7 +25,7 @@ type transportFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
|
||||
func TestRoleClientUsesSelectedNativeCredential(t *testing.T) {
|
||||
t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ")
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
|
||||
@@ -41,7 +40,7 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
providerErr := errors.New("provider echoed synthetic-secret-value")
|
||||
providerErr := errors.New("transport stopped")
|
||||
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" {
|
||||
@@ -62,10 +61,6 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
|
||||
if err == nil || calls != 1 {
|
||||
t.Fatalf("error = %v, calls = %d", err, calls)
|
||||
}
|
||||
masked := runtime.RedactError(context.Background(), providerErr)
|
||||
if strings.Contains(masked, "synthetic-secret-value") {
|
||||
t.Fatal("credential not redacted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
|
||||
@@ -282,10 +277,4 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
secrets := runtime.ConfiguredSecrets(context.Background())
|
||||
for _, value := range []string{"backend-secret-value", "profile-secret-value"} {
|
||||
if !slices.Contains(secrets, value) {
|
||||
t.Fatalf("missing credential masking value")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
@@ -20,12 +19,10 @@ import (
|
||||
|
||||
// Runtime owns the captured engine shared by all processing clients.
|
||||
type Runtime struct {
|
||||
Config config.Config
|
||||
Engine *pk.Engine
|
||||
SourcePaths []string
|
||||
nextCallID atomic.Uint64
|
||||
credentialSourcesOnce sync.Once
|
||||
credentialEnvNames map[string]struct{}
|
||||
Config config.Config
|
||||
Engine *pk.Engine
|
||||
SourcePaths []string
|
||||
nextCallID atomic.Uint64
|
||||
}
|
||||
|
||||
type AssetError struct{ Err error }
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
)
|
||||
|
||||
// ConfiguredSecrets obtains only values referenced by captured credential metadata.
|
||||
// It never selects a credential or treats an absent value as an error.
|
||||
func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string {
|
||||
if r == nil || r.Engine == nil {
|
||||
return nil
|
||||
}
|
||||
r.credentialSourcesOnce.Do(func() {
|
||||
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
|
||||
})
|
||||
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
|
||||
read := func(name string) {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
secrets = append(secrets, value)
|
||||
// PromptKit trims the value before sending it to the provider.
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
|
||||
secrets = append(secrets, trimmed)
|
||||
}
|
||||
}
|
||||
}
|
||||
for name := range r.credentialEnvNames {
|
||||
read(name)
|
||||
}
|
||||
if len(additionalEnvNames) > 0 {
|
||||
seen := make(map[string]struct{}, len(additionalEnvNames))
|
||||
for _, name := range additionalEnvNames {
|
||||
if _, known := r.credentialEnvNames[name]; known {
|
||||
continue
|
||||
}
|
||||
if _, duplicate := seen[name]; name == "" || duplicate {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
read(name)
|
||||
}
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
|
||||
names := make(map[string]struct{})
|
||||
add := func(name *string) {
|
||||
if name != nil && *name != "" {
|
||||
names[*name] = struct{}{}
|
||||
}
|
||||
}
|
||||
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
|
||||
for _, id := range ids {
|
||||
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
|
||||
add(profile.APIKeyEnv)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ids, err := r.Engine.ListBackends(ctx); err == nil {
|
||||
for _, id := range ids {
|
||||
if backend, err := r.Engine.InspectBackend(ctx, id); err == nil {
|
||||
add(backend.APIKeyEnv)
|
||||
}
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error())
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
|
||||
var empty *Runtime
|
||||
if empty.ConfiguredSecrets(context.Background()) != nil {
|
||||
t.Fatal("nil runtime has secrets")
|
||||
}
|
||||
t.Setenv("SELECTED_KEY", "selected-secret")
|
||||
t.Setenv("OTHER_KEY", "unselected-secret")
|
||||
t.Setenv("UNRELATED_KEY", "unrelated-secret")
|
||||
dir := t.TempDir()
|
||||
for name, data := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := NewRuntime(context.Background(), loaded, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
|
||||
seen[value] = true
|
||||
}
|
||||
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
|
||||
t.Fatalf("unexpected resolved backend secrets: %v", seen)
|
||||
}
|
||||
t.Setenv("SELECTED_KEY", "rotated-secret")
|
||||
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
|
||||
slices.Sort(values)
|
||||
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
|
||||
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,6 @@ type DebugInput struct {
|
||||
RunID string
|
||||
Metadata diagnostics.InvocationMetadata
|
||||
EffectiveConfig []byte
|
||||
Secrets []string
|
||||
}
|
||||
|
||||
type Prepared struct {
|
||||
@@ -85,7 +84,7 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
|
||||
}
|
||||
result := &Prepared{Output: session}
|
||||
if input.Debug != nil {
|
||||
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig, input.Debug.Secrets...)
|
||||
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, cancelOutput(err)
|
||||
|
||||
@@ -130,17 +130,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
var secrets []string
|
||||
if source, ok := req.LLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = source.ConfiguredSecrets(ctx)
|
||||
}
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||
secrets,
|
||||
),
|
||||
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
)
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
secrets []string
|
||||
responses []StructuredCorrectionSet
|
||||
err error
|
||||
completion contracts.StructuredCompletionResponse
|
||||
@@ -50,8 +49,6 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
|
||||
|
||||
type captureDiagnosticsWriter struct {
|
||||
err error
|
||||
lastStage string
|
||||
@@ -311,21 +308,16 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
primarySecret := "proposal-primary-secret"
|
||||
validationSecret := "proposal-validation-secret"
|
||||
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}},
|
||||
},
|
||||
}
|
||||
t.Setenv("PRIMARY_SECRET", primarySecret)
|
||||
t.Setenv("VALIDATION_SECRET", validationSecret)
|
||||
client.secrets = []string{primarySecret, validationSecret}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}
|
||||
req.Variables = map[string]string{"SectionJSON": "section text"}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -340,11 +332,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read artifact %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||
if !json.Valid(raw) {
|
||||
t.Fatalf("invalid JSON artifact %q: %s", path, raw)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,6 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
}
|
||||
current := original
|
||||
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
|
||||
findings := repairFindings(original.ProposalIndex, rejected)
|
||||
redactor := redaction.New(repairSecrets(ctx, input))
|
||||
for i := range findings {
|
||||
findings[i].Explanation = redactor.String(findings[i].Explanation)
|
||||
}
|
||||
return findings
|
||||
}
|
||||
findings := safeFindings(rejected)
|
||||
findings := repairFindings(original.ProposalIndex, rejected)
|
||||
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
|
||||
if !ok {
|
||||
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
|
||||
@@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input))
|
||||
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
|
||||
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
|
||||
if writeErr == nil {
|
||||
record.Artifacts = artifacts
|
||||
@@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
findings = safeFindings(validated.rejected)
|
||||
findings = repairFindings(original.ProposalIndex, validated.rejected)
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
@@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
return finish(nil)
|
||||
}
|
||||
|
||||
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
|
||||
var secrets []string
|
||||
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
|
||||
if source, ok := client.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
|
||||
}
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ type repairClient struct {
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
secrets []string
|
||||
}
|
||||
|
||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
@@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}
|
||||
|
||||
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
@@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
@@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
if a == "" || a == b {
|
||||
t.Fatalf("artifact paths: %q %q", a, b)
|
||||
}
|
||||
for _, path := range []string{a, b} {
|
||||
for i, path := range []string{a, b} {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
|
||||
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
|
||||
if !json.Valid(data) {
|
||||
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
|
||||
}
|
||||
var request proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
|
||||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
|
||||
t.Fatalf("repair request for proposal %d: %+v", i, request)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
|
||||
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
|
||||
}
|
||||
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), "validation-secret") {
|
||||
t.Fatalf("credential in artifact: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
|
||||
func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
|
||||
if interactionWriter == nil {
|
||||
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
|
||||
if source, ok := input.ValidationLLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
|
||||
}
|
||||
}
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
@@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
|
||||
}
|
||||
|
||||
type llmDiagnosticsWriterAdapter struct {
|
||||
dir string
|
||||
secrets func() []string
|
||||
dir string
|
||||
}
|
||||
|
||||
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
|
||||
var secrets []string
|
||||
if a.secrets != nil {
|
||||
secrets = a.secrets()
|
||||
}
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return validators.InteractionArtifacts{}, err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
func ptrConfig(c config.Config) *config.Config { return &c }
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []validators.LLMValidationResponse
|
||||
secrets []string
|
||||
preparedSecret string
|
||||
err error
|
||||
calls int
|
||||
responses []validators.LLMValidationResponse
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
|
||||
|
||||
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
f.calls++
|
||||
if f.preparedSecret != "" {
|
||||
f.secrets = append(f.secrets, f.preparedSecret)
|
||||
f.preparedSecret = ""
|
||||
}
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
@@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
primarySecret := "runner-primary-secret"
|
||||
validationSecret := "runner-validation-secret"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
|
||||
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
client.secrets = []string{primarySecret}
|
||||
client.preparedSecret = validationSecret
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
|
||||
ValidationDiagnosticsDir: diagDir,
|
||||
})
|
||||
@@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
|
||||
if !json.Valid(raw) {
|
||||
t.Fatalf("invalid diagnostic %q: %s", path, raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read decision diagnostic: %v", readErr)
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in diagnostics")
|
||||
if !strings.Contains(string(raw), "supported by context") {
|
||||
t.Fatalf("missing decision evidence: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user