eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reuse credential discovery and diagnostic redactors

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 20:37:52 +00:00
parent bb63c2cb10
commit 1fd95abe95
9 changed files with 103 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
internal/cli/process_output.go
View File

@@ -17,9 +17,9 @@ import (
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
reporter := inv.Reporter()
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
redact := func(err error) string { return loaded.Runtime.RedactError(inv.Context, err) }
safe := redaction.New(secrets)
for _, warning := range prepared.Prepared.Warnings {
_ = reporter.Warn("Debug artifact: " + redact(warning))
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
}
completed := time.Now().UTC()
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
@@ -52,9 +52,8 @@ func finishProcess(inv commandkit.Invocation, input processInput, loaded *proces
}
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil {
return commandkit.Failure(redact(result)).WithCause(result)
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
}
safe := redaction.New(secrets)
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range prepared.Prepared.Output.ReportPaths() {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))

8
internal/framework/llm/diagnostics.go
View File

@@ -23,13 +23,13 @@ type InteractionArtifacts struct {
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
type DiagnosticsWriter struct {
dir string
secrets []string
dir string
redactor redaction.Redactor
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir, secrets: append([]string(nil), secrets...)}
return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,7 +82,7 @@ func sanitizeStageName(stage string) string {
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
redacted, err := redaction.New(w.secrets).JSON(payload)
redacted, err := w.redactor.JSON(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
}

11
internal/framework/llm/runtime.go
View File

@@ -8,6 +8,7 @@ import (
"net/http"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -19,10 +20,12 @@ import (
// Runtime owns the captured engine shared by all processing clients.
type Runtime struct {
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
credentialSourcesOnce sync.Once
credentialEnvNames map[string]struct{}
}
type AssetError struct{ Err error }

49
internal/framework/llm/secrets.go
View File

@@ -14,16 +14,45 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
if r == nil || r.Engine == nil {
return nil
}
ctx = context.WithoutCancel(ctx)
r.credentialSourcesOnce.Do(func() {
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
})
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
read := func(name string) {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
for name := range r.credentialEnvNames {
read(name)
}
if len(additionalEnvNames) > 0 {
seen := make(map[string]struct{}, len(additionalEnvNames))
for _, name := range additionalEnvNames {
if _, known := r.credentialEnvNames[name]; known {
continue
}
if _, duplicate := seen[name]; name == "" || duplicate {
continue
}
seen[name] = struct{}{}
read(name)
}
}
return secrets
}
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
names := make(map[string]struct{})
add := func(name *string) {
if name != nil && *name != "" {
names[*name] = struct{}{}
}
}
for _, name := range additionalEnvNames {
add(&name)
}
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
for _, id := range ids {
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
@@ -38,17 +67,7 @@ func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...s
}
}
}
secrets := make([]string, 0, len(names))
for name := range names {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
return secrets
return names
}
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {

7
internal/framework/llm/secrets_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"slices"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -42,4 +43,10 @@ func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
t.Setenv("SELECTED_KEY", "rotated-secret")
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
slices.Sort(values)
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
}
}

36
internal/framework/proposal_generation/generate.go
View File

@@ -99,24 +99,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
}
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
}
}
var (
response StructuredCorrectionSet
completion contracts.StructuredCompletionResponse
@@ -153,6 +135,24 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
var writer InteractionDiagnosticsWriter
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
}
}
if writer != nil {
var diagnosticErr error
artifacts, diagnosticErr = writer.WriteInteraction(

33
internal/framework/runner/runner.go
View File

@@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
operations := make([]outcomes.Operation, 0)
filtered := make([]outcomes.FilteredProposal, 0)
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
var interactionWriter *llmDiagnosticsWriterAdapter
for validatorOrder, validator := range input.Validators {
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
var diagnosticsWriter validators.InteractionDiagnosticsWriter
if input.DiagnosticsDir != "" {
var secrets []string
if source, ok := input.ValidationLLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
secrets,
),
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
if source, ok := input.ValidationLLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
}
}
diagnosticsWriter = interactionWriter
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
@@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
type llmDiagnosticsWriterAdapter struct {
writer *llm.DiagnosticsWriter
dir string
secrets func() []string
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
if a == nil || a.writer == nil {
return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
var secrets []string
if a.secrets != nil {
secrets = a.secrets()
}
art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}

16
internal/framework/runner/runner_test.go
View File

@@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
err error
calls int
responses []validators.LLMValidationResponse
secrets []string
preparedSecret string
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
@@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
_ = ctx
_ = req
f.calls++
if f.preparedSecret != "" {
f.secrets = append(f.secrets, f.preparedSecret)
f.preparedSecret = ""
}
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
@@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret, validationSecret}
client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{