Reuse credential discovery and diagnostic redactors

internal/framework/llm/diagnostics.go

@@ -23,13 +23,13 @@ type InteractionArtifacts struct {
 
 // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
 type DiagnosticsWriter struct {
-	dir     string
-	secrets []string
+	dir      string
+	redactor redaction.Redactor
 }
 
 // NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
 func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
-	return &DiagnosticsWriter{dir: dir, secrets: append([]string(nil), secrets...)}
+	return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
 }
 
 // WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,7 +82,7 @@ func sanitizeStageName(stage string) string {
 }
 
 func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
-	redacted, err := redaction.New(w.secrets).JSON(payload)
+	redacted, err := w.redactor.JSON(payload)
 	if err != nil {
 		return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
 	}

internal/framework/llm/runtime.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"strings"
+	"sync"
 	"sync/atomic"
 
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -19,10 +20,12 @@ import (
 
 // Runtime owns the captured engine shared by all processing clients.
 type Runtime struct {
-	Config      config.Config
-	Engine      *pk.Engine
-	SourcePaths []string
-	nextCallID  atomic.Uint64
+	Config                config.Config
+	Engine                *pk.Engine
+	SourcePaths           []string
+	nextCallID            atomic.Uint64
+	credentialSourcesOnce sync.Once
+	credentialEnvNames    map[string]struct{}
 }
 
 type AssetError struct{ Err error }

internal/framework/llm/secrets.go

@@ -14,16 +14,45 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
 	if r == nil || r.Engine == nil {
 		return nil
 	}
-	ctx = context.WithoutCancel(ctx)
+	r.credentialSourcesOnce.Do(func() {
+		r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
+	})
+	secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
+	read := func(name string) {
+		if value := os.Getenv(name); value != "" {
+			secrets = append(secrets, value)
+			// PromptKit trims the value before sending it to the provider.
+			if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
+				secrets = append(secrets, trimmed)
+			}
+		}
+	}
+	for name := range r.credentialEnvNames {
+		read(name)
+	}
+	if len(additionalEnvNames) > 0 {
+		seen := make(map[string]struct{}, len(additionalEnvNames))
+		for _, name := range additionalEnvNames {
+			if _, known := r.credentialEnvNames[name]; known {
+				continue
+			}
+			if _, duplicate := seen[name]; name == "" || duplicate {
+				continue
+			}
+			seen[name] = struct{}{}
+			read(name)
+		}
+	}
+	return secrets
+}
+
+func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
 	names := make(map[string]struct{})
 	add := func(name *string) {
 		if name != nil && *name != "" {
 			names[*name] = struct{}{}
 		}
 	}
-	for _, name := range additionalEnvNames {
-		add(&name)
-	}
 	if ids, err := r.Engine.ListProfiles(ctx); err == nil {
 		for _, id := range ids {
 			if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
@@ -38,17 +67,7 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
 			}
 		}
 	}
-	secrets := make([]string, 0, len(names))
-	for name := range names {
-		if value := os.Getenv(name); value != "" {
-			secrets = append(secrets, value)
-			// PromptKit trims the value before sending it to the provider.
-			if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
-				secrets = append(secrets, trimmed)
-			}
-		}
-	}
-	return secrets
+	return names
 }
 
 func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {

internal/framework/llm/secrets_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -42,4 +43,10 @@ func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
 	if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
 		t.Fatalf("unexpected resolved backend secrets: %v", seen)
 	}
+	t.Setenv("SELECTED_KEY", "rotated-secret")
+	values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
+	slices.Sort(values)
+	if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
+		t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
+	}
 }

internal/framework/proposal_generation/generate.go

@@ -99,24 +99,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
 		stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
 	}
 
-	var writer InteractionDiagnosticsWriter
-	if req.DiagnosticsWriter != nil {
-		writer = req.DiagnosticsWriter
-	} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
-		var secrets []string
-		if source, ok := req.LLMClient.(interface {
-			ConfiguredSecrets(context.Context) []string
-		}); ok {
-			secrets = source.ConfiguredSecrets(ctx)
-		}
-		writer = diagnosticsWriterAdapter{
-			writer: llm.NewDiagnosticsWriter(
-				filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
-				secrets,
-			),
-		}
-	}
-
 	var (
 		response   StructuredCorrectionSet
 		completion contracts.StructuredCompletionResponse
@@ -153,6 +135,24 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
 	}
 	requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
 
+	var writer InteractionDiagnosticsWriter
+	if req.DiagnosticsWriter != nil {
+		writer = req.DiagnosticsWriter
+	} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
+		var secrets []string
+		if source, ok := req.LLMClient.(interface {
+			ConfiguredSecrets(context.Context) []string
+		}); ok {
+			secrets = source.ConfiguredSecrets(ctx)
+		}
+		writer = diagnosticsWriterAdapter{
+			writer: llm.NewDiagnosticsWriter(
+				filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
+				secrets,
+			),
+		}
+	}
+
 	if writer != nil {
 		var diagnosticErr error
 		artifacts, diagnosticErr = writer.WriteInteraction(

internal/framework/runner/runner.go

@@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
 	operations := make([]outcomes.Operation, 0)
 	filtered := make([]outcomes.FilteredProposal, 0)
 	eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
+	var interactionWriter *llmDiagnosticsWriterAdapter
 
 	for validatorOrder, validator := range input.Validators {
 		if err := ctx.Err(); err != nil {
 			return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
 		}
 		var diagnosticsWriter validators.InteractionDiagnosticsWriter
-		if input.DiagnosticsDir != "" {
-			var secrets []string
-			if source, ok := input.ValidationLLMClient.(interface {
-				ConfiguredSecrets(context.Context) []string
-			}); ok {
-				secrets = source.ConfiguredSecrets(ctx)
-			}
-			diagnosticsWriter = &llmDiagnosticsWriterAdapter{
-				writer: llm.NewDiagnosticsWriter(
-					filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
-					secrets,
-				),
+		_, directLLMValidator := validator.(*validators.LLMBackedValidator)
+		if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
+			if interactionWriter == nil {
+				interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
+				if source, ok := input.ValidationLLMClient.(interface {
+					ConfiguredSecrets(context.Context) []string
+				}); ok {
+					interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
+				}
 			}
+			diagnosticsWriter = interactionWriter
 		}
 
 		vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
@@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
 }
 
 type llmDiagnosticsWriterAdapter struct {
-	writer *llm.DiagnosticsWriter
+	dir     string
+	secrets func() []string
 }
 
 func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
-	if a == nil || a.writer == nil {
-		return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
+	var secrets []string
+	if a.secrets != nil {
+		secrets = a.secrets()
 	}
-	art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
+	art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
 	if err != nil {
 		return validators.InteractionArtifacts{}, err
 	}

internal/framework/runner/runner_test.go

@@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
 func ptrConfig(c config.Config) *config.Config { return &c }
 
 type fakeStructuredClient struct {
-	responses []validators.LLMValidationResponse
-	secrets   []string
-	err       error
-	calls     int
+	responses      []validators.LLMValidationResponse
+	secrets        []string
+	preparedSecret string
+	err            error
+	calls          int
 }
 
 func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
@@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
 	_ = ctx
 	_ = req
 	f.calls++
+	if f.preparedSecret != "" {
+		f.secrets = append(f.secrets, f.preparedSecret)
+		f.preparedSecret = ""
+	}
 	if f.err != nil {
 		return contracts.StructuredCompletionResponse{}, f.err
 	}
@@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
 	validationSecret := "runner-validation-secret"
 	client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
 	llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
-	client.secrets = []string{primarySecret, validationSecret}
+	client.secrets = []string{primarySecret}
+	client.preparedSecret = validationSecret
 	cfg := runnerTestConfig()
 	diagDir := t.TempDir()
 	r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{