Unify structured call completion and retain typed failures
This commit is contained in:
@@ -46,7 +46,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
cause := errors.New("underlying transport error")
|
||||
failed := &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed", Cause: cause}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
@@ -67,6 +68,10 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
|
||||
t.Fatalf("batch operations: %+v", result.Operations)
|
||||
}
|
||||
operation := result.Operations[1]
|
||||
if operation.FailureClass != outcomes.FailureRecoverable || !errors.Is(operation.Cause, cause) {
|
||||
t.Fatalf("validation lost typed failure facts: %+v", operation)
|
||||
}
|
||||
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package validators
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -103,6 +102,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
operations = append(operations, outcomes.Operation{
|
||||
Reference: operationRef(outcomes.OperationValidationInput, nil),
|
||||
FailureClass: outcomes.FailureRecoverable,
|
||||
FailureReason: outcomes.ReasonValidationInputTooLarge,
|
||||
ProposalIndices: indices,
|
||||
})
|
||||
@@ -173,17 +173,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
if err != nil {
|
||||
artifacts := writeInteraction(nil, err)
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(err, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
}
|
||||
if operation.FailureReason == "" {
|
||||
operation.FailureReason = outcomes.ReasonInternalFailure
|
||||
}
|
||||
structuredoutput.RecordFailure(&operation, err)
|
||||
operation.ProposalIndices = batchProposalIndices(batch.Items)
|
||||
operations = append(operations, operation)
|
||||
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
|
||||
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable")
|
||||
for i := range decisions {
|
||||
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
}
|
||||
@@ -196,6 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
reconciled := reconcileLLMResponse(batch.Items, response)
|
||||
artifacts := writeInteraction(&reconciled, nil)
|
||||
if len(reconciled.UnavailableIndices) > 0 {
|
||||
operation.FailureClass = outcomes.FailureRecoverable
|
||||
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
|
||||
operation.ProposalIndices = reconciled.UnavailableIndices
|
||||
} else {
|
||||
|
||||
@@ -191,7 +191,7 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
||||
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
|
||||
Reference in New Issue
Block a user