Unify optional interaction capture and warnings
This commit is contained in:
@@ -123,10 +123,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
var response LLMValidationResponse
|
||||
var completion StructuredCompletionResponse
|
||||
promptMetadata := validatorPromptMetadata(v.validatorType)
|
||||
coordinates := structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt}
|
||||
completion, err = req.LLMClient.CompleteStructured(ctx, StructuredCompletionRequest{
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables, Schema: responseschema.ValidatorDecisionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: req.StageIndex, StageID: req.StageID, Role: "validation", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex, BatchIndex: &batch.BatchIndex, ValidatorName: v.name, ProposalIndex: req.RepairProposalIndex, RepairAttempt: req.RepairAttempt},
|
||||
Context: coordinates,
|
||||
}, &response)
|
||||
batchIndex := batch.BatchIndex
|
||||
operation := outcomes.Operation{
|
||||
@@ -134,9 +135,9 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
Counts: completion.Execution,
|
||||
}
|
||||
|
||||
writeInteraction := func(reconciled *validationReconciliation, callErr error) InteractionArtifacts {
|
||||
if req.DiagnosticsWriter == nil {
|
||||
return InteractionArtifacts{}
|
||||
writeInteraction := func(reconciled *validationReconciliation, callErr error) structuredoutput.InteractionArtifacts {
|
||||
if req.Interactions == nil {
|
||||
return structuredoutput.InteractionArtifacts{}
|
||||
}
|
||||
stageInstance := req.ModuleInstance
|
||||
if req.SectionIndex != nil {
|
||||
@@ -151,10 +152,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if reconciled != nil {
|
||||
responsePayload["reconciliation"] = *reconciled
|
||||
}
|
||||
var diagnosticErr error
|
||||
artifacts, diagnosticErr := req.DiagnosticsWriter.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{
|
||||
artifacts, _ := req.Interactions.WriteInteraction(structuredoutput.Interaction{
|
||||
Name: stage, Context: coordinates, RequestMetadata: map[string]any{
|
||||
"stage_index": req.StageIndex, "stage_id": req.StageID, "module_key": req.ModuleKey,
|
||||
"validator_name": v.name,
|
||||
"validator_type": v.validatorType,
|
||||
@@ -164,17 +163,14 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
"prompt": completion.Prompt,
|
||||
"response_schema": responseSchema.DiagnosticsMap(),
|
||||
},
|
||||
completion.Prepared,
|
||||
responsePayload,
|
||||
errPayload(callErr),
|
||||
)
|
||||
if diagnosticErr != nil && req.ReportDiagnosticError != nil {
|
||||
req.ReportDiagnosticError(fmt.Errorf("validation interaction %s: %w", stage, diagnosticErr))
|
||||
}
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: responsePayload,
|
||||
ErrorPayload: errPayload(callErr),
|
||||
})
|
||||
return artifacts
|
||||
}
|
||||
if err != nil {
|
||||
writeInteraction(nil, err)
|
||||
artifacts := writeInteraction(nil, err)
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(err, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
@@ -185,7 +181,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
operation.ProposalIndices = batchProposalIndices(batch.Items)
|
||||
operations = append(operations, operation)
|
||||
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
|
||||
llmDecisions = append(llmDecisions, rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")...)
|
||||
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
|
||||
for i := range decisions {
|
||||
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
}
|
||||
llmDecisions = append(llmDecisions, decisions...)
|
||||
continue
|
||||
}
|
||||
return partialResult(), fmt.Errorf("LLM validator %q completion failed: %w", v.name, err)
|
||||
|
||||
@@ -30,13 +30,10 @@ type captureValidationDiagnosticsWriter struct {
|
||||
lastResponsePayload any
|
||||
}
|
||||
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
_ = stage
|
||||
_ = requestPayload
|
||||
_ = errorPayload
|
||||
w.lastRequestMetadata = requestMetadata
|
||||
w.lastResponsePayload = responsePayload
|
||||
return InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
|
||||
func (w *captureValidationDiagnosticsWriter) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
w.lastRequestMetadata = interaction.RequestMetadata
|
||||
w.lastResponsePayload = interaction.ResponsePayload
|
||||
return structuredoutput.InteractionArtifacts{ResponsePayloadPath: w.path}, w.err
|
||||
}
|
||||
|
||||
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
|
||||
@@ -237,7 +234,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
v, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "actors"), mk(18, "gestures", "figures")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
req.Interactions = writer
|
||||
result, err := v.Validate(context.Background(), req)
|
||||
if err != nil || len(client.calls) != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d err=%v", result, len(client.calls), err)
|
||||
@@ -343,7 +340,7 @@ func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
}
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = writer
|
||||
req.Interactions = writer
|
||||
_, err = v.Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("validate error: %v", err)
|
||||
@@ -399,12 +396,11 @@ func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
sink := &captureValidationDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
result, err := validator.Validate(context.Background(), req)
|
||||
if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
|
||||
if err != nil || sink.lastRequestMetadata == nil || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
|
||||
t.Fatalf("debug failure changed outcome: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -29,28 +30,26 @@ const (
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
type Request struct {
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
||||
StageIndex *int `json:"stage_index,omitempty"`
|
||||
StageID *string `json:"stage_id,omitempty"`
|
||||
IndependentEvidence bool `json:"-"`
|
||||
RepairProposalIndex *int `json:"repair_proposal_index,omitempty"`
|
||||
RepairAttempt *int `json:"repair_attempt,omitempty"`
|
||||
SectionIndex *int `json:"section_index,omitempty"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *SectionLookup `json:"-"`
|
||||
Previews map[int]proposals.SegmentPreviewResult `json:"-"`
|
||||
CandidateProposal []proposals.EnrichedCorrectionProposal `json:"candidate_proposals"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
GlossaryView *GlossaryView `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
Interactions structuredoutput.InteractionSink `json:"-"`
|
||||
}
|
||||
|
||||
// Decision is one validator decision for one proposal index.
|
||||
@@ -81,17 +80,6 @@ type Result struct {
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// Validator is the deterministic runtime validator interface.
|
||||
type Validator interface {
|
||||
Name() string
|
||||
|
||||
Reference in New Issue
Block a user