Remove application credential substitution
This commit is contained in:
@@ -7,8 +7,6 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -108,15 +106,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
}
|
||||
current := original
|
||||
safeFindings := func(rejected []ValidatorRejectedChange) []proposal_generation.RepairFinding {
|
||||
findings := repairFindings(original.ProposalIndex, rejected)
|
||||
redactor := redaction.New(repairSecrets(ctx, input))
|
||||
for i := range findings {
|
||||
findings[i].Explanation = redactor.String(findings[i].Explanation)
|
||||
}
|
||||
return findings
|
||||
}
|
||||
findings := safeFindings(rejected)
|
||||
findings := repairFindings(original.ProposalIndex, rejected)
|
||||
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
|
||||
if !ok {
|
||||
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
|
||||
@@ -175,7 +165,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
if err != nil {
|
||||
failure = map[string]string{"error": err.Error()}
|
||||
}
|
||||
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName), repairSecrets(ctx, input))
|
||||
writer := llm.NewDiagnosticsWriter(filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName))
|
||||
artifacts, writeErr := writer.WriteInteraction(name, map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": proposalIndex, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": findings}, completion.Prepared, map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": record.Revision, "decisions": record.Decisions, "rejections": validated.rejected}, failure)
|
||||
if writeErr == nil {
|
||||
record.Artifacts = artifacts
|
||||
@@ -254,7 +244,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
findings = safeFindings(validated.rejected)
|
||||
findings = repairFindings(original.ProposalIndex, validated.rejected)
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
@@ -278,15 +268,3 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
return finish(nil)
|
||||
}
|
||||
|
||||
func repairSecrets(ctx context.Context, input validateSectionCandidatesInput) []string {
|
||||
var secrets []string
|
||||
for _, client := range []contracts.StructuredLLMClient{input.Spec.RepairClient, input.ValidationLLMClient} {
|
||||
if source, ok := client.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = append(secrets, source.ConfiguredSecrets(ctx)...)
|
||||
}
|
||||
}
|
||||
return secrets
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ type repairClient struct {
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
secrets []string
|
||||
}
|
||||
|
||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
@@ -47,8 +46,6 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}
|
||||
|
||||
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
@@ -288,8 +285,8 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
@@ -310,13 +307,21 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
if a == "" || a == b {
|
||||
t.Fatalf("artifact paths: %q %q", a, b)
|
||||
}
|
||||
for _, path := range []string{a, b} {
|
||||
for i, path := range []string{a, b} {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), "synthetic-secret") || !strings.Contains(string(data), "[REDACTED]") {
|
||||
t.Fatalf("unredacted %s: %s", filepath.Base(path), data)
|
||||
if !json.Valid(data) {
|
||||
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
|
||||
}
|
||||
var request proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
|
||||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
|
||||
t.Fatalf("repair request for proposal %d: %+v", i, request)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -355,28 +360,6 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
|
||||
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(client.calls[0].Variables["RepairRequestJSON"], "validation-secret") {
|
||||
t.Fatalf("credential in feedback: %s", client.calls[0].Variables["RepairRequestJSON"])
|
||||
}
|
||||
data, err := os.ReadFile(branches[0].Attempts[0].Artifacts.RequestMetadataPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(data), "validation-secret") {
|
||||
t.Fatalf("credential in artifact: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
|
||||
func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
@@ -408,7 +391,7 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -704,11 +704,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
|
||||
if interactionWriter == nil {
|
||||
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
|
||||
if source, ok := input.ValidationLLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
|
||||
}
|
||||
}
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
@@ -992,16 +987,11 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
|
||||
}
|
||||
|
||||
type llmDiagnosticsWriterAdapter struct {
|
||||
dir string
|
||||
secrets func() []string
|
||||
dir string
|
||||
}
|
||||
|
||||
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
|
||||
var secrets []string
|
||||
if a.secrets != nil {
|
||||
secrets = a.secrets()
|
||||
}
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return validators.InteractionArtifacts{}, err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -920,23 +921,15 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
func ptrConfig(c config.Config) *config.Config { return &c }
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []validators.LLMValidationResponse
|
||||
secrets []string
|
||||
preparedSecret string
|
||||
err error
|
||||
calls int
|
||||
responses []validators.LLMValidationResponse
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
|
||||
|
||||
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
f.calls++
|
||||
if f.preparedSecret != "" {
|
||||
f.secrets = append(f.secrets, f.preparedSecret)
|
||||
f.preparedSecret = ""
|
||||
}
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
@@ -1159,13 +1152,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
primarySecret := "runner-primary-secret"
|
||||
validationSecret := "runner-validation-secret"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: validationSecret}}}}}
|
||||
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
client.secrets = []string{primarySecret}
|
||||
client.preparedSecret = validationSecret
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
@@ -1175,7 +1164,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
}})
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
|
||||
ValidationDiagnosticsDir: diagDir,
|
||||
})
|
||||
@@ -1197,8 +1186,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read diagnostic %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), primarySecret) || strings.Contains(string(raw), validationSecret) {
|
||||
t.Fatalf("configured secret leaked in diagnostics %q: %s", path, string(raw))
|
||||
if !json.Valid(raw) {
|
||||
t.Fatalf("invalid diagnostic %q: %s", path, raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1206,8 +1195,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
if readErr != nil {
|
||||
t.Fatalf("read decision diagnostic: %v", readErr)
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in diagnostics")
|
||||
if !strings.Contains(string(raw), "supported by context") {
|
||||
t.Fatalf("missing decision evidence: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user