Reuse credential discovery and diagnostic redactors
This commit is contained in:
@@ -587,25 +587,24 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
filtered := make([]outcomes.FilteredProposal, 0)
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
var interactionWriter *llmDiagnosticsWriterAdapter
|
||||
|
||||
for validatorOrder, validator := range input.Validators {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||
if input.DiagnosticsDir != "" {
|
||||
var secrets []string
|
||||
if source, ok := input.ValidationLLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = source.ConfiguredSecrets(ctx)
|
||||
}
|
||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||
secrets,
|
||||
),
|
||||
_, directLLMValidator := validator.(*validators.LLMBackedValidator)
|
||||
if input.DiagnosticsDir != "" && (directLLMValidator || validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked) {
|
||||
if interactionWriter == nil {
|
||||
interactionWriter = &llmDiagnosticsWriterAdapter{dir: filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName)}
|
||||
if source, ok := input.ValidationLLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
interactionWriter.secrets = func() []string { return source.ConfiguredSecrets(ctx) }
|
||||
}
|
||||
}
|
||||
diagnosticsWriter = interactionWriter
|
||||
}
|
||||
|
||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
@@ -857,14 +856,16 @@ func cloneTranscript(t *schema.Transcript) *schema.Transcript {
|
||||
}
|
||||
|
||||
type llmDiagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
dir string
|
||||
secrets func() []string
|
||||
}
|
||||
|
||||
func (a *llmDiagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (validators.InteractionArtifacts, error) {
|
||||
if a == nil || a.writer == nil {
|
||||
return validators.InteractionArtifacts{}, fmt.Errorf("diagnostics writer is not configured")
|
||||
var secrets []string
|
||||
if a.secrets != nil {
|
||||
secrets = a.secrets()
|
||||
}
|
||||
art, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
art, err := llm.NewDiagnosticsWriter(a.dir, secrets).WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return validators.InteractionArtifacts{}, err
|
||||
}
|
||||
|
||||
@@ -913,10 +913,11 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
func ptrConfig(c config.Config) *config.Config { return &c }
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []validators.LLMValidationResponse
|
||||
secrets []string
|
||||
err error
|
||||
calls int
|
||||
responses []validators.LLMValidationResponse
|
||||
secrets []string
|
||||
preparedSecret string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
|
||||
@@ -925,6 +926,10 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
_ = ctx
|
||||
_ = req
|
||||
f.calls++
|
||||
if f.preparedSecret != "" {
|
||||
f.secrets = append(f.secrets, f.preparedSecret)
|
||||
f.preparedSecret = ""
|
||||
}
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
@@ -1152,7 +1157,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
validationSecret := "runner-validation-secret"
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
client.secrets = []string{primarySecret, validationSecret}
|
||||
client.secrets = []string{primarySecret}
|
||||
client.preparedSecret = validationSecret
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
|
||||
Reference in New Issue
Block a user