Reuse credential discovery and diagnostic redactors

This commit is contained in:
2026-11-10 20:37:52 +00:00
parent bb63c2cb10
commit 1fd95abe95
9 changed files with 103 additions and 66 deletions

View File

@@ -487,6 +487,8 @@ covered by R2. Inspection retains its native history commands.
## R14 — P3: Reuse Credential Source Discovery Across Diagnostics ## R14 — P3: Reuse Credential Source Discovery Across Diagnostics
**Status:** Completed. Credential source names are discovered once per runtime, with current values read at each masking boundary. Interaction and finalization redactors are reused, and deterministic validators receive no interaction writer. Tests cover credential changes and newly prepared names.
**Category** **Category**
Repeated catalog scans in Stages 4 and 8. Repeated catalog scans in Stages 4 and 8.

View File

@@ -17,9 +17,9 @@ import (
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome { func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
reporter := inv.Reporter() reporter := inv.Reporter()
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context) secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
redact := func(err error) string { return loaded.Runtime.RedactError(inv.Context, err) } safe := redaction.New(secrets)
for _, warning := range prepared.Prepared.Warnings { for _, warning := range prepared.Prepared.Warnings {
_ = reporter.Warn("Debug artifact: " + redact(warning)) _ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
} }
completed := time.Now().UTC() completed := time.Now().UTC()
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
@@ -52,9 +52,8 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
} }
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed) result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil { if result != nil {
return commandkit.Failure(redact(result)).WithCause(result) return commandkit.Failure(safe.String(result.Error())).WithCause(result)
} }
safe := redaction.New(secrets)
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot)) _ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range prepared.Prepared.Output.ReportPaths() { for _, path := range prepared.Prepared.Output.ReportPaths() {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path))) _ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))

View File

@@ -23,13 +23,13 @@ type InteractionArtifacts struct {
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts. // DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
type DiagnosticsWriter struct { type DiagnosticsWriter struct {
dir string dir string
secrets []string redactor redaction.Redactor
} }
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir. // NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter { func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir, secrets: append([]string(nil), secrets...)} return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
} }
// WriteInteraction writes request/response/error artifacts for a generic LLM // WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,7 +82,7 @@ func sanitizeStageName(stage string) string {
} }
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error { func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
redacted, err := redaction.New(w.secrets).JSON(payload) redacted, err := w.redactor.JSON(payload)
if err != nil { if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err) return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
} }

View File

@@ -8,6 +8,7 @@ import (
"net/http" "net/http"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -19,10 +20,12 @@ import (
// Runtime owns the captured engine shared by all processing clients. // Runtime owns the captured engine shared by all processing clients.
type Runtime struct { type Runtime struct {
Config config.Config Config config.Config
Engine *pk.Engine Engine *pk.Engine
SourcePaths []string SourcePaths []string
nextCallID atomic.Uint64 nextCallID atomic.Uint64
credentialSourcesOnce sync.Once
credentialEnvNames map[string]struct{}
} }
type AssetError struct{ Err error } type AssetError struct{ Err error }

View File

@@ -14,16 +14,45 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
if r == nil || r.Engine == nil { if r == nil || r.Engine == nil {
return nil return nil
} }
ctx = context.WithoutCancel(ctx) r.credentialSourcesOnce.Do(func() {
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
})
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
read := func(name string) {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
for name := range r.credentialEnvNames {
read(name)
}
if len(additionalEnvNames) > 0 {
seen := make(map[string]struct{}, len(additionalEnvNames))
for _, name := range additionalEnvNames {
if _, known := r.credentialEnvNames[name]; known {
continue
}
if _, duplicate := seen[name]; name == "" || duplicate {
continue
}
seen[name] = struct{}{}
read(name)
}
}
return secrets
}
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
names := make(map[string]struct{}) names := make(map[string]struct{})
add := func(name *string) { add := func(name *string) {
if name != nil && *name != "" { if name != nil && *name != "" {
names[*name] = struct{}{} names[*name] = struct{}{}
} }
} }
for _, name := range additionalEnvNames {
add(&name)
}
if ids, err := r.Engine.ListProfiles(ctx); err == nil { if ids, err := r.Engine.ListProfiles(ctx); err == nil {
for _, id := range ids { for _, id := range ids {
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil { if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
@@ -38,17 +67,7 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
} }
} }
} }
secrets := make([]string, 0, len(names)) return names
for name := range names {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
return secrets
} }
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string { func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"testing" "testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -42,4 +43,10 @@ func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] { if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen) t.Fatalf("unexpected resolved backend secrets: %v", seen)
} }
t.Setenv("SELECTED_KEY", "rotated-secret")
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
slices.Sort(values)
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
}
} }

View File

@@ -99,24 +99,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section)) stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
} }
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
}
}
var ( var (
response StructuredCorrectionSet response StructuredCorrectionSet
completion contracts.StructuredCompletionResponse completion contracts.StructuredCompletionResponse
@@ -153,6 +135,24 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
} }
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap() requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
}
}
if writer != nil { if writer != nil {
var diagnosticErr error var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction( artifacts, diagnosticErr = writer.WriteInteraction(

View File

@@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
operations := make([]outcomes.Operation, 0) operations := make([]outcomes.Operation, 0)
filtered := make([]outcomes.FilteredProposal, 0) filtered := make([]outcomes.FilteredProposal, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators { for validatorOrder, validator := range input.Validators {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
} }
var diagnosticsWriter validators.InteractionDiagnosticsWriter var diagnosticsWriter validators.InteractionDiagnosticsWriter
if input.DiagnosticsDir != "" { _, directLLMValidator := validator.(*validators.LLMBackedValidator)
var secrets []string if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if source, ok := input.ValidationLLMClient.(interface { if interactionWriter == nil {
ConfiguredSecrets(context.Context) []string interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
}); ok { if source, ok := input.ValidationLLMClient.(interface {
secrets = source.ConfiguredSecrets(ctx) ConfiguredSecrets(context.Context) []string
} }); ok {
diagnosticsWriter = &llmDiagnosticsWriterAdapter{ interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
writer: llm.NewDiagnosticsWriter( }
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
secrets,
),
} }
diagnosticsWriter = interactionWriter
} }
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
@@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
} }
type llmDiagnosticsWriterAdapter struct { type llmDiagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter dir string
secrets func() []string
} }
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) { func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
if a == nil || a.writer == nil { var secrets []string
return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured") if a.secrets != nil {
secrets = a.secrets()
} }
art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload) art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil { if err != nil {
return validators.InteractionArtifacts{}, err return validators.InteractionArtifacts{}, err
} }

View File

@@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
func ptrConfig(c config.Config) *config.Config { return &c } func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct { type fakeStructuredClient struct {
responses []validators.LLMValidationResponse responses []validators.LLMValidationResponse
secrets []string secrets []string
err error preparedSecret string
calls int err error
calls int
} }
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets } func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
@@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
_ = ctx _ = ctx
_ = req _ = req
f.calls++ f.calls++
if f.preparedSecret != "" {
f.secrets = append(f.secrets, f.preparedSecret)
f.preparedSecret = ""
}
if f.err != nil { if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err return contracts.StructuredCompletionResponse{}, f.err
} }
@@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret" validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}} client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret, validationSecret} client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
cfg := runnerTestConfig() cfg := runnerTestConfig()
diagDir := t.TempDir() diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{