Unify structured call completion and retain typed failures

This commit is contained in:
2026-11-16 14:27:54 +00:00
parent e98e525b5c
commit c2c8050bc8
18 changed files with 313 additions and 186 deletions

View File

@@ -46,7 +46,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
cause := errors.New("underlying transport error")
failed := &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed", Cause: cause}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
@@ -67,6 +68,10 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
t.Fatalf("batch operations: %+v", result.Operations)
}
operation := result.Operations[1]
if operation.FailureClass != outcomes.FailureRecoverable || !errors.Is(operation.Cause, cause) {
t.Fatalf("validation lost typed failure facts: %+v", operation)
}
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
t.Fatal(err)
}

View File

@@ -3,7 +3,6 @@ package validators
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
@@ -103,6 +102,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
operations = append(operations, outcomes.Operation{
Reference: operationRef(outcomes.OperationValidationInput, nil),
FailureClass: outcomes.FailureRecoverable,
FailureReason: outcomes.ReasonValidationInputTooLarge,
ProposalIndices: indices,
})
@@ -173,17 +173,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
if err != nil {
artifacts := writeInteraction(nil, err)
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
}
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
structuredoutput.RecordFailure(&operation, err)
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
if operation.FailureClass == outcomes.FailureRecoverable {
decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable")
for i := range decisions {
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
@@ -196,6 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
reconciled := reconcileLLMResponse(batch.Items, response)
artifacts := writeInteraction(&reconciled, nil)
if len(reconciled.UnavailableIndices) > 0 {
operation.FailureClass = outcomes.FailureRecoverable
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = reconciled.UnavailableIndices
} else {

View File

@@ -191,7 +191,7 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
}
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client