Reuse credential discovery and diagnostic redactors

docs/roadmap/remediation.md

@@ -487,6 +487,8 @@ covered by R2. Inspection retains its native history commands.
 
 ## R14 — P3: Reuse Credential Source Discovery Across Diagnostics
 
+**Status:** Completed. Credential source names are discovered once per runtime, with current values read at each masking boundary. Interaction and finalization redactors are reused, and deterministic validators receive no interaction writer. Tests cover credential changes and newly prepared names.
+
 **Category**
 
 Repeated catalog scans in Stages 4 and 8.

internal/cli/process_output.go

@@ -17,9 +17,9 @@ import (
 func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
 	reporter := inv.Reporter()
 	secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
-	redact := func(err error) string { return loaded.Runtime.RedactError(inv.Context, err) }
+	safe := redaction.New(secrets)
 	for _, warning := range prepared.Prepared.Warnings {
-		_ = reporter.Warn("Debug artifact: " + redact(warning))
+		_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
 	}
 	completed := time.Now().UTC()
 	snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
@@ -52,9 +52,8 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
 	}
 	result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
 	if result != nil {
-		return commandkit.Failure(redact(result)).WithCause(result)
+		return commandkit.Failure(safe.String(result.Error())).WithCause(result)
 	}
-	safe := redaction.New(secrets)
 	_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
 	for _, path := range prepared.Prepared.Output.ReportPaths() {
 		_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))

internal/framework/llm/diagnostics.go

@@ -23,13 +23,13 @@ type InteractionArtifacts struct {
 
 // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
 type DiagnosticsWriter struct {
-	dir     string
-	secrets []string
+	dir      string
+	redactor redaction.Redactor
 }
 
 // NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
 func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
-	return &DiagnosticsWriter{dir: dir, secrets: append([]string(nil), secrets...)}
+	return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
 }
 
 // WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,7 +82,7 @@ func sanitizeStageName(stage string) string {
 }
 
 func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
-	redacted, err := redaction.New(w.secrets).JSON(payload)
+	redacted, err := w.redactor.JSON(payload)
 	if err != nil {
 		return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
 	}

internal/framework/llm/runtime.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"strings"
+	"sync"
 	"sync/atomic"
 
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -19,10 +20,12 @@ import (
 
 // Runtime owns the captured engine shared by all processing clients.
 type Runtime struct {
-	Config      config.Config
-	Engine      *pk.Engine
-	SourcePaths []string
-	nextCallID  atomic.Uint64
+	Config                config.Config
+	Engine                *pk.Engine
+	SourcePaths           []string
+	nextCallID            atomic.Uint64
+	credentialSourcesOnce sync.Once
+	credentialEnvNames    map[string]struct{}
 }
 
 type AssetError struct{ Err error }

internal/framework/llm/secrets.go

@@ -14,16 +14,45 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
 	if r == nil || r.Engine == nil {
 		return nil
 	}
-	ctx = context.WithoutCancel(ctx)
+	r.credentialSourcesOnce.Do(func() {
+		r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
+	})
+	secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
+	read := func(name string) {
+		if value := os.Getenv(name); value != "" {
+			secrets = append(secrets, value)
+			// PromptKit trims the value before sending it to the provider.
+			if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
+				secrets = append(secrets, trimmed)
+			}
+		}
+	}
+	for name := range r.credentialEnvNames {
+		read(name)
+	}
+	if len(additionalEnvNames) > 0 {
+		seen := make(map[string]struct{}, len(additionalEnvNames))
+		for _, name := range additionalEnvNames {
+			if _, known := r.credentialEnvNames[name]; known {
+				continue
+			}
+			if _, duplicate := seen[name]; name == "" || duplicate {
+				continue
+			}
+			seen[name] = struct{}{}
+			read(name)
+		}
+	}
+	return secrets
+}
+
+func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
 	names := make(map[string]struct{})
 	add := func(name *string) {
 		if name != nil && *name != "" {
 			names[*name] = struct{}{}
 		}
 	}
-	for _, name := range additionalEnvNames {
-		add(&name)
-	}
 	if ids, err := r.Engine.ListProfiles(ctx); err == nil {
 		for _, id := range ids {
 			if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
@@ -38,17 +67,7 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
 			}
 		}
 	}
-	secrets := make([]string, 0, len(names))
-	for name := range names {
-		if value := os.Getenv(name); value != "" {
-			secrets = append(secrets, value)
-			// PromptKit trims the value before sending it to the provider.
-			if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
-				secrets = append(secrets, trimmed)
-			}
-		}
-	}
-	return secrets
+	return names
 }
 
 func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {

internal/framework/llm/secrets_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -42,4 +43,10 @@ func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
 	if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
 		t.Fatalf("unexpected resolved backend secrets: %v", seen)
 	}
+	t.Setenv("SELECTED_KEY", "rotated-secret")
+	values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
+	slices.Sort(values)
+	if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
+		t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
+	}
 }

internal/framework/proposal_generation/generate.go

@@ -99,24 +99,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
 		stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
 	}
 
-	var writer InteractionDiagnosticsWriter
-	if req.DiagnosticsWriter != nil {
-		writer = req.DiagnosticsWriter
-	} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
-		var secrets []string
-		if source, ok := req.LLMClient.(interface {
-			ConfiguredSecrets(context.Context) []string
-		}); ok {
-			secrets = source.ConfiguredSecrets(ctx)
-		}
-		writer = diagnosticsWriterAdapter{
-			writer: llm.NewDiagnosticsWriter(
-				filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
-				secrets,
-			),
-		}
-	}
-
 	var (
 		response   StructuredCorrectionSet
 		completion contracts.StructuredCompletionResponse
@@ -153,6 +135,24 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
 	}
 	requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
 
+	var writer InteractionDiagnosticsWriter
+	if req.DiagnosticsWriter != nil {
+		writer = req.DiagnosticsWriter
+	} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
+		var secrets []string
+		if source, ok := req.LLMClient.(interface {
+			ConfiguredSecrets(context.Context) []string
+		}); ok {
+			secrets = source.ConfiguredSecrets(ctx)
+		}
+		writer = diagnosticsWriterAdapter{
+			writer: llm.NewDiagnosticsWriter(
+				filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
+				secrets,
+			),
+		}
+	}
+
 	if writer != nil {
 		var diagnosticErr error
 		artifacts, diagnosticErr = writer.WriteInteraction(

internal/framework/runner/runner.go

@@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
 	operations := make([]outcomes.Operation, 0)
 	filtered := make([]outcomes.FilteredProposal, 0)
 	eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
+	var interactionWriter *llmDiagnosticsWriterAdapter
 
 	for validatorOrder, validator := range input.Validators {
 		if err := ctx.Err(); err != nil {
 			return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
 		}
 		var diagnosticsWriter validators.InteractionDiagnosticsWriter
-		if input.DiagnosticsDir != "" {
-			var secrets []string
-			if source, ok := input.ValidationLLMClient.(interface {
-				ConfiguredSecrets(context.Context) []string
-			}); ok {
-				secrets = source.ConfiguredSecrets(ctx)
-			}
-			diagnosticsWriter = &llmDiagnosticsWriterAdapter{
-				writer: llm.NewDiagnosticsWriter(
-					filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
-					secrets,
-				),
+		_, directLLMValidator := validator.(*validators.LLMBackedValidator)
+		if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
+			if interactionWriter == nil {
+				interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
+				if source, ok := input.ValidationLLMClient.(interface {
+					ConfiguredSecrets(context.Context) []string
+				}); ok {
+					interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
+				}
 			}
+			diagnosticsWriter = interactionWriter
 		}
 
 		vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
@@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
 }
 
 type llmDiagnosticsWriterAdapter struct {
-	writer *llm.DiagnosticsWriter
+	dir     string
+	secrets func() []string
 }
 
 func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
-	if a == nil || a.writer == nil {
-		return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
+	var secrets []string
+	if a.secrets != nil {
+		secrets = a.secrets()
 	}
-	art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
+	art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
 	if err != nil {
 		return validators.InteractionArtifacts{}, err
 	}

internal/framework/runner/runner_test.go

@@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
 func ptrConfig(c config.Config) *config.Config { return &c }
 
 type fakeStructuredClient struct {
-	responses []validators.LLMValidationResponse
-	secrets   []string
-	err       error
-	calls     int
+	responses      []validators.LLMValidationResponse
+	secrets        []string
+	preparedSecret string
+	err            error
+	calls          int
 }
 
 func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
@@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
 	_ = ctx
 	_ = req
 	f.calls++
+	if f.preparedSecret != "" {
+		f.secrets = append(f.secrets, f.preparedSecret)
+		f.preparedSecret = ""
+	}
 	if f.err != nil {
 		return contracts.StructuredCompletionResponse{}, f.err
 	}
@@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
 	validationSecret := "runner-validation-secret"
 	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
 	llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
-	client.secrets = []string{primarySecret, validationSecret}
+	client.secrets = []string{primarySecret}
+	client.preparedSecret = validationSecret
 	cfg := runnerTestConfig()
 	diagDir := t.TempDir()
 	r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{