Remove application credential substitution

This commit is contained in:
2026-11-16 03:37:29 +00:00
parent e34672f7fe
commit ffe0e3491e
37 changed files with 217 additions and 729 deletions

View File

@@ -25,13 +25,11 @@ type CallHooks struct {
}
type RoleClient struct {
runtime *Runtime
role string
profileID string
settings config.LLM
hooks CallHooks
credentialMu sync.Mutex
credentialNames map[string]struct{}
runtime *Runtime
role string
profileID string
settings config.LLM
hooks CallHooks
}
var _ structuredoutput.Client = (*RoleClient)(nil)
@@ -152,14 +150,6 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
var details pk.PreparedRun
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
details = prepared
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
c.credentialNames = make(map[string]struct{})
}
c.credentialNames[name] = struct{}{}
c.credentialMu.Unlock()
}
response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model
@@ -311,17 +301,6 @@ func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}
// ConfiguredSecrets supplies artifact redaction with the engine's referenced credentials.
func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
c.credentialMu.Lock()
names := make([]string, 0, len(c.credentialNames))
for name := range c.credentialNames {
names = append(names, name)
}
c.credentialMu.Unlock()
return c.runtime.ConfiguredSecrets(ctx, names...)
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages))
for i, message := range messages {

View File

@@ -1,13 +1,12 @@
package llm
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
var stageSanitizer = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`)
@@ -23,13 +22,12 @@ type InteractionArtifacts struct {
// DiagnosticsWriter writes generic machine-readable LLM interaction artifacts.
type DiagnosticsWriter struct {
dir string
redactor redaction.Redactor
dir string
}
// NewDiagnosticsWriter creates a diagnostics writer rooted at dir.
func NewDiagnosticsWriter(dir string, secrets []string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir, redactor: redaction.New(secrets)}
func NewDiagnosticsWriter(dir string) *DiagnosticsWriter {
return &DiagnosticsWriter{dir: dir}
}
// WriteInteraction writes request/response/error artifacts for a generic LLM
@@ -82,12 +80,12 @@ func sanitizeStageName(stage string) string {
}
func (w *DiagnosticsWriter) writeJSON(path string, payload any) error {
redacted, err := w.redactor.JSON(payload)
encoded, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal json artifact %q: %w", filepath.Base(path), err)
}
redacted = append(redacted, '\n')
if err := os.WriteFile(path, redacted, 0o644); err != nil {
encoded = append(encoded, '\n')
if err := os.WriteFile(path, encoded, 0o644); err != nil {
return fmt.Errorf("write json artifact %q: %w", filepath.Base(path), err)
}
return nil

View File

@@ -4,13 +4,12 @@ import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
dir := t.TempDir()
writer := NewDiagnosticsWriter(dir, nil)
writer := NewDiagnosticsWriter(dir)
artifacts, err := writer.WriteInteraction(
"validation:batch-1",
@@ -31,45 +30,6 @@ func TestDiagnosticsWriterProducesValidJSONArtifacts(t *testing.T) {
assertValidJSONFile(t, artifacts.ResponsePayloadPath)
}
func TestDiagnosticsWriterRedactsSecrets(t *testing.T) {
dir := t.TempDir()
secret := "top-secret-token"
writer := NewDiagnosticsWriter(dir, []string{secret})
artifacts, err := writer.WriteInteraction(
"module glossary / request",
map[string]any{"authorization": "Bearer " + secret},
map[string]any{"api_key": secret, "nested": map[string]any{"value": secret}},
map[string]any{"echo": secret},
map[string]any{"error": "failed with " + secret},
)
if err != nil {
t.Fatalf("WriteInteraction: %v", err)
}
for _, path := range []string{
artifacts.RequestMetadataPath,
artifacts.RequestPayloadPath,
artifacts.ResponsePayloadPath,
artifacts.ErrorPayloadPath,
} {
raw, readErr := os.ReadFile(path)
if readErr != nil {
t.Fatalf("read %s: %v", path, readErr)
}
content := string(raw)
if strings.Contains(content, secret) {
t.Fatalf("secret leaked in %s: %s", filepath.Base(path), content)
}
if !strings.Contains(content, "[REDACTED]") {
t.Fatalf("redaction marker missing in %s: %s", filepath.Base(path), content)
}
if !json.Valid(raw) {
t.Fatalf("invalid json in %s", filepath.Base(path))
}
}
}
func assertValidJSONFile(t *testing.T, path string) {
t.Helper()
raw, err := os.ReadFile(path)

View File

@@ -9,7 +9,6 @@ import (
"net/http"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -26,7 +25,7 @@ type transportFunc func(*http.Request) (*http.Response, error)
func (f transportFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
func TestRoleClientUsesSelectedNativeCredential(t *testing.T) {
t.Setenv("AUDITA_TEST_KEY", " synthetic-secret-value\t ")
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte("version: 3\n"), 0644); err != nil {
@@ -41,7 +40,7 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
t.Fatal(err)
}
calls := 0
providerErr := errors.New("provider echoed synthetic-secret-value")
providerErr := errors.New("transport stopped")
httpClient := &http.Client{Transport: transportFunc(func(r *http.Request) (*http.Response, error) {
calls++
if r.Header.Get("Authorization") != "Bearer synthetic-secret-value" {
@@ -62,10 +61,6 @@ func TestRoleClientUsesNativeTargetAndRedactsCredential(t *testing.T) {
if err == nil || calls != 1 {
t.Fatalf("error = %v, calls = %d", err, calls)
}
masked := runtime.RedactError(context.Background(), providerErr)
if strings.Contains(masked, "synthetic-secret-value") {
t.Fatal("credential not redacted")
}
}
func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
@@ -282,10 +277,4 @@ func TestNativeCredentialSelectionAndOptionalAbsence(t *testing.T) {
}
})
}
secrets := runtime.ConfiguredSecrets(context.Background())
for _, value := range []string{"backend-secret-value", "profile-secret-value"} {
if !slices.Contains(secrets, value) {
t.Fatalf("missing credential masking value")
}
}
}

View File

@@ -8,7 +8,6 @@ import (
"net/http"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -20,12 +19,10 @@ import (
// Runtime owns the captured engine shared by all processing clients.
type Runtime struct {
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
credentialSourcesOnce sync.Once
credentialEnvNames map[string]struct{}
Config config.Config
Engine *pk.Engine
SourcePaths []string
nextCallID atomic.Uint64
}
type AssetError struct{ Err error }

View File

@@ -1,78 +0,0 @@
package llm
import (
"context"
"os"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
)
// ConfiguredSecrets obtains only values referenced by captured credential metadata.
// It never selects a credential or treats an absent value as an error.
func (r *Runtime) ConfiguredSecrets(ctx context.Context, additionalEnvNames ...string) []string {
if r == nil || r.Engine == nil {
return nil
}
r.credentialSourcesOnce.Do(func() {
r.credentialEnvNames = r.discoverCredentialEnvNames(context.WithoutCancel(ctx))
})
secrets := make([]string, 0, len(r.credentialEnvNames)+len(additionalEnvNames))
read := func(name string) {
if value := os.Getenv(name); value != "" {
secrets = append(secrets, value)
// PromptKit trims the value before sending it to the provider.
if trimmed := strings.TrimSpace(value); trimmed != "" && trimmed != value {
secrets = append(secrets, trimmed)
}
}
}
for name := range r.credentialEnvNames {
read(name)
}
if len(additionalEnvNames) > 0 {
seen := make(map[string]struct{}, len(additionalEnvNames))
for _, name := range additionalEnvNames {
if _, known := r.credentialEnvNames[name]; known {
continue
}
if _, duplicate := seen[name]; name == "" || duplicate {
continue
}
seen[name] = struct{}{}
read(name)
}
}
return secrets
}
func (r *Runtime) discoverCredentialEnvNames(ctx context.Context) map[string]struct{} {
names := make(map[string]struct{})
add := func(name *string) {
if name != nil && *name != "" {
names[*name] = struct{}{}
}
}
if ids, err := r.Engine.ListProfiles(ctx); err == nil {
for _, id := range ids {
if profile, err := r.Engine.InspectProfile(ctx, id); err == nil {
add(profile.APIKeyEnv)
}
}
}
if ids, err := r.Engine.ListBackends(ctx); err == nil {
for _, id := range ids {
if backend, err := r.Engine.InspectBackend(ctx, id); err == nil {
add(backend.APIKeyEnv)
}
}
}
return names
}
func (r *Runtime) RedactError(ctx context.Context, err error, additionalEnvNames ...string) string {
if err == nil {
return ""
}
return redaction.New(r.ConfiguredSecrets(ctx, additionalEnvNames...)).String(err.Error())
}

View File

@@ -1,52 +0,0 @@
package llm
import (
"context"
"os"
"path/filepath"
"slices"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
t.Setenv("SELECTED_KEY", "rotated-secret")
values := runtime.ConfiguredSecrets(context.Background(), "SELECTED_KEY", "UNRELATED_KEY", "UNRELATED_KEY")
slices.Sort(values)
if !slices.Equal(values, []string{"rotated-secret", "unrelated-secret", "unselected-secret"}) {
t.Fatalf("values did not reflect the new credential and prepared source: %v", values)
}
}

View File

@@ -28,7 +28,6 @@ type DebugInput struct {
RunID string
Metadata diagnostics.InvocationMetadata
EffectiveConfig []byte
Secrets []string
}
type Prepared struct {
@@ -85,7 +84,7 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
}
result := &Prepared{Output: session}
if input.Debug != nil {
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig, input.Debug.Secrets...)
result.Debug, result.Warnings = session.PrepareDebug(input.Debug.WorkDir, input.Debug.RunID, input.Debug.Metadata, input.Debug.EffectiveConfig)
}
if err := ctx.Err(); err != nil {
return nil, cancelOutput(err)

View File

@@ -130,17 +130,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if req.DiagnosticsWriter != nil {
writer = req.DiagnosticsWriter
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
var secrets []string
if source, ok := req.LLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = source.ConfiguredSecrets(ctx)
}
writer = diagnosticsWriterAdapter{
writer: llm.NewDiagnosticsWriter(
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
secrets,
),
writer: llm.NewDiagnosticsWriter(filepath.Join(req.DiagnosticsDir, req.ModuleInstance)),
}
}

View File

@@ -19,7 +19,6 @@ import (
)
type fakeStructuredClient struct {
secrets []string
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
@@ -50,8 +49,6 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
return response, nil
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
type captureDiagnosticsWriter struct {
err error
lastStage string
@@ -311,21 +308,16 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
}
}
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "proposal-primary-secret"
validationSecret := "proposal-validation-secret"
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "source text", CorrectedText: "safe"}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
client.secrets = []string{primarySecret, validationSecret}
req := defaultRequest(t)
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}
req.Variables = map[string]string{"SectionJSON": "section text"}
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -340,11 +332,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read artifact %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("artifact leaked configured secret in %q: %s", path, string(raw))
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
if !json.Valid(raw) {
t.Fatalf("invalid JSON artifact %q: %s", path, raw)
}
}

View File

@@ -7,8 +7,6 @@ import (
"path/filepath"
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
}
current := original
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
findings := repairFindings(original.ProposalIndex, rejected)
redactor := redaction.New(repairSecrets(ctx, input))
for i := range findings {
findings[i].Explanation = redactor.String(findings[i].Explanation)
}
return findings
}
findings := safeFindings(rejected)
findings := repairFindings(original.ProposalIndex, rejected)
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
if !ok {
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
@@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
if err != nil {
failure = map[string]string{"error": err.Error()}
}
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input))
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
if writeErr == nil {
record.Artifacts = artifacts
@@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return nil, nil
}
if actionable(original.ProposalIndex, validated.rejected) {
findings = safeFindings(validated.rejected)
findings = repairFindings(original.ProposalIndex, validated.rejected)
return buildRequest()
}
branch.Outcome = "unavailable"
@@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
return finish(nil)
}
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
var secrets []string
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
if source, ok := client.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
}
}
return secrets
}

View File

@@ -31,7 +31,6 @@ type repairClient struct {
outputs []string
calls []structuredoutput.Request
err error
secrets []string
}
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
@@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
}
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
@@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
}
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
func TestRepairArtifactsAreUnique(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
@@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for _, path := range []string{a, b} {
for i, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
if !json.Valid(data) {
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
}
var request proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
t.Fatal(err)
}
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
t.Fatalf("repair request for proposal %d: %+v", i, request)
}
}
}
@@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
}
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
}
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(data), "validation-secret") {
t.Fatalf("credential in artifact: %s", data)
}
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
@@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)

View File

@@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
if interactionWriter == nil {
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
if source, ok := input.ValidationLLMClient.(interface {
ConfiguredSecrets(context.Context) []string
}); ok {
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
}
}
diagnosticsWriter = interactionWriter
}
@@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
}
type llmDiagnosticsWriterAdapter struct {
dir string
secrets func() []string
dir string
}
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
var secrets []string
if a.secrets != nil {
secrets = a.secrets()
}
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
if err != nil {
return validators.InteractionArtifacts{}, err
}

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
@@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
preparedSecret string
err error
calls int
responses []validators.LLMValidationResponse
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
f.calls++
if f.preparedSecret != "" {
f.secrets = append(f.secrets, f.preparedSecret)
f.preparedSecret = ""
}
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
@@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}
}
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
primarySecret := "runner-primary-secret"
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
client.secrets = []string{primarySecret}
client.preparedSecret = validationSecret
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
@@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
@@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read diagnostic %q: %v", path, readErr)
}
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
if !json.Valid(raw) {
t.Fatalf("invalid diagnostic %q: %s", path, raw)
}
}
@@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
if readErr != nil {
t.Fatalf("read decision diagnostic: %v", readErr)
}
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatalf("expected redaction marker in diagnostics")
if !strings.Contains(string(raw), "supported by context") {
t.Fatalf("missing decision evidence: %s", raw)
}
}