From 1fd95abe95fd050aa1058d4b6e1b6be36215aeca Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 10 Nov 2026 20:37:52 +0000 Subject: [PATCH] Reuse credential discovery and diagnostic redactors --- docs/roadmap/remediation.md | 2 + internal/cli/process_output.go | 7 ++- internal/framework/llm/diagnostics.go | 8 +-- internal/framework/llm/runtime.go | 11 +++-- internal/framework/llm/secrets.go | 49 +++++++++++++------ internal/framework/llm/secrets_test.go | 7 +++ .../framework/proposal_generation/generate.go | 36 +++++++------- internal/framework/runner/runner.go | 33 +++++++------ internal/framework/runner/runner_test.go | 16 ++++-- 9 files changed, 103 insertions(+), 66 deletions(-) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 30a19ea..d4890d3 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -487,6 +487,8 @@ covered by R2. Inspection retains its native history commands. ## R14 — P3: Reuse Credential Source Discovery Across Diagnostics +**Status:** Completed. Credential source names are discovered once per runtime, with current values read at each masking boundary. Interaction and finalization redactors are reused, and deterministic validators receive no interaction writer. Tests cover credential changes and newly prepared names. + **Category** Repeated catalog scans in Stages 4 and 8. diff --git a/internal/cli/process_output.go b/internal/cli/process_output.go index 494451f..aa548a0 100644 --- a/internal/cli/process_output.go +++ b/internal/cli/process_output.go @@ -17,9 +17,9 @@ import ( func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome { reporter := inv.Reporter() secrets := loaded.Runtime.ConfiguredSecrets(inv.Context) - redact := func(err error) string { return loaded.Runtime.RedactError(inv.Context, err) } + safe := redaction.New(secrets) for _, warning := range prepared.Prepared.Warnings { - _ = reporter.Warn("Debug artifact: " + redact(warning)) + _ = reporter.Warn("Debug artifact: " + safe.String(warning.Error())) } completed := time.Now().UTC() snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed @@ -52,9 +52,8 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces } result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed) if result != nil { - return commandkit.Failure(redact(result)).WithCause(result) + return commandkit.Failure(safe.String(result.Error())).WithCause(result) } - safe := redaction.New(secrets) _ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot)) for _, path := range prepared.Prepared.Output.ReportPaths() { _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path))) diff --git a/internal/framework/llm/diagnostics.go b/internal/framework/llm/diagnostics.go index eb5dbf1..037b783 100644 --- a/internal/framework/llm/diagnostics.go +++ b/internal/framework/llm/diagnostics.go @@ -23,13 +23,13 @@ type InteractionArtifacts struct { // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts. type DiagnosticsWriter struct { - dir string - secrets []string + dir string + redactor redaction.Redactor } // NewDiagnosticsWriter creates a diagnostics writer rooted at dir. func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter { - return &DiagnosticsWriter{dir: dir, secrets: append([]string(nil), secrets...)} + return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)} } // WriteInteraction writes request/response/error artifacts for a generic LLM @@ -82,7 +82,7 @@ func sanitizeStageName(stage string) string { } func (w *DiagnosticsWriter) writeJSON(path string, payload any) error { - redacted, err := redaction.New(w.secrets).JSON(payload) + redacted, err := w.redactor.JSON(payload) if err != nil { return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err) } diff --git a/internal/framework/llm/runtime.go b/internal/framework/llm/runtime.go index b63da72..6a95568 100644 --- a/internal/framework/llm/runtime.go +++ b/internal/framework/llm/runtime.go @@ -8,6 +8,7 @@ import ( "net/http" "path/filepath" "strings" + "sync" "sync/atomic" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -19,10 +20,12 @@ import ( // Runtime owns the captured engine shared by all processing clients. type Runtime struct { - Config config.Config - Engine *pk.Engine - SourcePaths []string - nextCallID atomic.Uint64 + Config config.Config + Engine *pk.Engine + SourcePaths []string + nextCallID atomic.Uint64 + credentialSourcesOnce sync.Once + credentialEnvNames map[string]struct{} } type AssetError struct{ Err error } diff --git a/internal/framework/llm/secrets.go b/internal/framework/llm/secrets.go index 84f9a39..53a13f1 100644 --- a/internal/framework/llm/secrets.go +++ b/internal/framework/llm/secrets.go @@ -14,16 +14,45 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s if r == nil || r.Engine == nil { return nil } - ctx = context.WithoutCancel(ctx) + r.credentialSourcesOnce.Do(func() { + r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx)) + }) + secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames)) + read := func(name string) { + if value := os.Getenv(name); value != "" { + secrets = append(secrets, value) + // PromptKit trims the value before sending it to the provider. + if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value { + secrets = append(secrets, trimmed) + } + } + } + for name := range r.credentialEnvNames { + read(name) + } + if len(additionalEnvNames) > 0 { + seen := make(map[string]struct{}, len(additionalEnvNames)) + for _, name := range additionalEnvNames { + if _, known := r.credentialEnvNames[name]; known { + continue + } + if _, duplicate := seen[name]; name == "" || duplicate { + continue + } + seen[name] = struct{}{} + read(name) + } + } + return secrets +} + +func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} { names := make(map[string]struct{}) add := func(name *string) { if name != nil && *name != "" { names[*name] = struct{}{} } } - for _, name := range additionalEnvNames { - add(&name) - } if ids, err := r.Engine.ListProfiles(ctx); err == nil { for _, id := range ids { if profile, err := r.Engine.InspectProfile(ctx, id); err == nil { @@ -38,17 +67,7 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s } } } - secrets := make([]string, 0, len(names)) - for name := range names { - if value := os.Getenv(name); value != "" { - secrets = append(secrets, value) - // PromptKit trims the value before sending it to the provider. - if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value { - secrets = append(secrets, trimmed) - } - } - } - return secrets + return names } func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string { diff --git a/internal/framework/llm/secrets_test.go b/internal/framework/llm/secrets_test.go index fb8e5f3..b25d6df 100644 --- a/internal/framework/llm/secrets_test.go +++ b/internal/framework/llm/secrets_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "slices" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -42,4 +43,10 @@ func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) { if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] { t.Fatalf("unexpected resolved backend secrets: %v", seen) } + t.Setenv("SELECTED_KEY", "rotated-secret") + values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY") + slices.Sort(values) + if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) { + t.Fatalf("values did not reflect the new credential and prepared source: %v", values) + } } diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index f1e01fd..9ed1ecb 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -99,24 +99,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section)) } - var writer InteractionDiagnosticsWriter - if req.DiagnosticsWriter != nil { - writer = req.DiagnosticsWriter - } else if strings.TrimSpace(req.DiagnosticsDir) != "" { - var secrets []string - if source, ok := req.LLMClient.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - secrets = source.ConfiguredSecrets(ctx) - } - writer = diagnosticsWriterAdapter{ - writer: llm.NewDiagnosticsWriter( - filepath.Join(req.DiagnosticsDir, req.ModuleInstance), - secrets, - ), - } - } - var ( response StructuredCorrectionSet completion contracts.StructuredCompletionResponse @@ -153,6 +135,24 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() + var writer InteractionDiagnosticsWriter + if req.DiagnosticsWriter != nil { + writer = req.DiagnosticsWriter + } else if strings.TrimSpace(req.DiagnosticsDir) != "" { + var secrets []string + if source, ok := req.LLMClient.(interface { + ConfiguredSecrets(context.Context) []string + }); ok { + secrets = source.ConfiguredSecrets(ctx) + } + writer = diagnosticsWriterAdapter{ + writer: llm.NewDiagnosticsWriter( + filepath.Join(req.DiagnosticsDir, req.ModuleInstance), + secrets, + ), + } + } + if writer != nil { var diagnosticErr error artifacts, diagnosticErr = writer.WriteInteraction( diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index 5452def..e913458 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida operations := make([]outcomes.Operation, 0) filtered := make([]outcomes.FilteredProposal, 0) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) + var interactionWriter *llmDiagnosticsWriterAdapter for validatorOrder, validator := range input.Validators { if err := ctx.Err(); err != nil { return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err } var diagnosticsWriter validators.InteractionDiagnosticsWriter - if input.DiagnosticsDir != "" { - var secrets []string - if source, ok := input.ValidationLLMClient.(interface { - ConfiguredSecrets(context.Context) []string - }); ok { - secrets = source.ConfiguredSecrets(ctx) - } - diagnosticsWriter = &llmDiagnosticsWriterAdapter{ - writer: llm.NewDiagnosticsWriter( - filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), - secrets, - ), + _, directLLMValidator := validator.(*validators.LLMBackedValidator) + if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) { + if interactionWriter == nil { + interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)} + if source, ok := input.ValidationLLMClient.(interface { + ConfiguredSecrets(context.Context) []string + }); ok { + interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) } + } } + diagnosticsWriter = interactionWriter } vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ @@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript { } type llmDiagnosticsWriterAdapter struct { - writer *llm.DiagnosticsWriter + dir string + secrets func() []string } func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { - if a == nil || a.writer == nil { - return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured") + var secrets []string + if a.secrets != nil { + secrets = a.secrets() } - art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) + art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) if err != nil { return validators.InteractionArtifacts{}, err } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 7251f4a..313bc83 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { func ptrConfig(c config.Config) *config.Config { return &c } type fakeStructuredClient struct { - responses []validators.LLMValidationResponse - secrets []string - err error - calls int + responses []validators.LLMValidationResponse + secrets []string + preparedSecret string + err error + calls int } func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } @@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr _ = ctx _ = req f.calls++ + if f.preparedSecret != "" { + f.secrets = append(f.secrets, f.preparedSecret) + f.preparedSecret = "" + } if f.err != nil { return contracts.StructuredCompletionResponse{}, f.err } @@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) { validationSecret := "runner-validation-secret" client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) - client.secrets = []string{primarySecret, validationSecret} + client.secrets = []string{primarySecret} + client.preparedSecret = validationSecret cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{