Unify structured call completion and retain typed failures
This commit is contained in:
@@ -1206,6 +1206,30 @@ func TestSubprocessInfeasibleSectionTargetIsAnInputFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessMissingCredentialIdentifiesRequiredEnvironmentVariable(t *testing.T) {
|
||||
f := newProcessFixture(t, "success", "[grammar]")
|
||||
f.provider.secret = ""
|
||||
resources := filepath.Join(f.dir, "prompts.yml")
|
||||
body, err := os.ReadFile(resources)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body = bytes.Replace(body, []byte(" - id: proposal\n"), []byte(" - id: proposal\n api_key_required: true\n api_key_env: AUDITA_TEST_PROVIDER_KEY\n"), 1)
|
||||
if err := os.WriteFile(resources, body, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
if code != 1 || stdout != "" || f.provider.callCount() != 0 ||
|
||||
!strings.Contains(stderr, "AUDITA_TEST_PROVIDER_KEY") || !strings.Contains(stderr, `proposal profile "proposal"`) {
|
||||
t.Fatalf("missing credential: status=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||
}
|
||||
document := f.errors(t)
|
||||
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 ||
|
||||
document.Errors[0].Reason != "setup_failure" || strings.Contains(document.Errors[0].Message, "AUDITA_TEST_PROVIDER_KEY") {
|
||||
t.Fatalf("expected compact setup failure in ordinary report: %+v", document)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessUnknownProfileFailsBeforeModelDispatchAndOutput(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
code, stdout, stderr := f.run(t, true, "--proposal-profile", "unknown")
|
||||
|
||||
@@ -12,10 +12,14 @@ profiles and effective budgets. Clients share engine-local backend admission.
|
||||
The adapter uses ordinary `Run` with `OperationTimeout` and `OnPrepared`, passes
|
||||
Audita's per-call schema and repair budget, and decodes only validated results.
|
||||
The operation budget includes preparation, admission, retries, and structural
|
||||
repair. `ErrOperationTimeout` remains recoverable even when wrapped by a
|
||||
preparation error. Parent cancellation remains distinct. The prepared notification
|
||||
repair. An expired parent context stops processing and retains any native cause.
|
||||
With an active parent, PromptKit's operation deadline remains recoverable even
|
||||
when a preparation error wraps it. A native transport timeout affects the HTTP
|
||||
attempt and is recorded as a transport failure. The prepared notification
|
||||
provides the actual request snapshot; failures before notification have no snapshot.
|
||||
PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md)
|
||||
PromptKit owns backend and credential selection. When a required credential is absent,
|
||||
the CLI identifies the role, profile, and environment variable using the native
|
||||
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)
|
||||
for failure behavior and the [configuration reference](../config.md) for sources.
|
||||
|
||||
When correction repair is enabled, the proposal call captures a native immutable
|
||||
@@ -26,7 +30,10 @@ for assessment. Follow-ups select the repair prompt, profile, and output schema
|
||||
directly; Audita does not rerender the original proposal or assemble replay messages.
|
||||
An opaque exchange in the structured-output contract keeps provider state outside
|
||||
modules and validators. The adapter shares request construction, notifications,
|
||||
observers, and result/failure finalization between ordinary and coordinated calls.
|
||||
observers, and once-only completion between ordinary and coordinated calls.
|
||||
Completion attaches terminal counts when available and records one logical finish.
|
||||
Prepared requests and terminal output travel in the returned response for an
|
||||
optional interaction artifact. Logical call notifications carry metadata.
|
||||
|
||||
Audita owns actionable rejection, revision decoding, withdrawal, current previews,
|
||||
and full revalidation. Accepted siblings remain independent. Requesting more work
|
||||
@@ -44,7 +51,11 @@ selected runs in a native `ExecutionGroup`. Proposal, repair, and nested validat
|
||||
clients explicitly share the invocation observer; history reuse adds no extra run.
|
||||
Native summaries deduplicate terminal totals by run ID and retain terminal coverage
|
||||
separately from usage presence and completeness. Logical calls and run membership
|
||||
can differ, especially on failures before execution. Ordinary
|
||||
reports use compact safe outcomes; request and response evidence is optional
|
||||
debug material. The runner preserves ordered pipeline traversal and deterministic
|
||||
section collection while canceling and joining peers on fatal errors.
|
||||
can differ, especially on failures before execution.
|
||||
|
||||
Processing operations retain the failure class, reason, and underlying error.
|
||||
When a client omits the reason, the operation uses `internal_failure` and keeps
|
||||
the original class and error chain. Ordinary reports use compact outcomes.
|
||||
Request and response evidence is optional debug material. The runner preserves
|
||||
ordered pipeline traversal and deterministic section collection while canceling
|
||||
and joining peers on fatal errors.
|
||||
|
||||
@@ -447,6 +447,14 @@ controlled budgets. Check setup class/cause and useful native information,
|
||||
plus ordinary/coordinator terminal and undelivered-evidence paths. Update
|
||||
[LLM runtime internals](../internal/llm-runtime.md).
|
||||
|
||||
**Complete.** Ordinary and coordinated calls share once-only completion,
|
||||
including prepared and undelivered evidence and native terminal counts.
|
||||
Proposal, validation, and repair retain typed failure facts. Independent
|
||||
parent, operation, and attempt deadlines produced the expected classifications.
|
||||
The CLI identifies a missing required credential by role, profile, and
|
||||
environment variable while the ordinary report retains the setup reason.
|
||||
The CLI build, full offline Go suite, and targeted race checks passed.
|
||||
|
||||
## Stage 13 — Use one proposal handoff
|
||||
|
||||
Audit #46.
|
||||
|
||||
@@ -68,7 +68,9 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
|
||||
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
|
||||
var cause *structuredoutput.Failure
|
||||
if errors.As(runErr, &cause) {
|
||||
failure.Reason = cause.Reason
|
||||
if cause.Reason != "" {
|
||||
failure.Reason = cause.Reason
|
||||
}
|
||||
} else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) {
|
||||
failure.Reason = outcomes.ReasonCanceled
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
)
|
||||
|
||||
// CallHooks receive one logical start and finish record and the execution
|
||||
// events for this call. Call records can contain transcript content.
|
||||
// events associated with the call.
|
||||
type CallHooks struct {
|
||||
OnCall func(structuredoutput.CallRecord)
|
||||
OnEvent pk.ExecutionObserver
|
||||
@@ -45,35 +45,25 @@ func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hoo
|
||||
}
|
||||
|
||||
type pendingCall struct {
|
||||
request pk.RunRequest
|
||||
finish func(*pk.RunResult, error) (structuredoutput.Response, error)
|
||||
request pk.RunRequest
|
||||
once sync.Once
|
||||
complete func(*pk.RunResult, error) (structuredoutput.Response, error)
|
||||
response structuredoutput.Response
|
||||
err error
|
||||
}
|
||||
|
||||
func (p *pendingCall) finish(result *pk.RunResult, err error) (structuredoutput.Response, error) {
|
||||
p.once.Do(func() {
|
||||
p.response, p.err = p.complete(result, err)
|
||||
})
|
||||
return p.response, p.err
|
||||
}
|
||||
|
||||
func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) {
|
||||
var response structuredoutput.Response
|
||||
var err error
|
||||
var finishHooks []func()
|
||||
ctx := parent
|
||||
var terminalMu sync.Mutex
|
||||
var mu sync.Mutex
|
||||
var terminal pk.ExecutionEvent
|
||||
terminalAvailable := false
|
||||
finishHooks = append(finishHooks, func() {
|
||||
terminalMu.Lock()
|
||||
if terminalAvailable {
|
||||
response.Execution = outcomes.ExecutionCounts{
|
||||
Available: true,
|
||||
ProviderAttempts: int(terminal.Attempts),
|
||||
TransientRetries: int(terminal.TransientRetries),
|
||||
StructuralRepairs: int(terminal.RepairCalls),
|
||||
RetriesExhausted: terminal.RetriesExhausted,
|
||||
}
|
||||
if terminal.StatusCode != 0 {
|
||||
status := terminal.StatusCode
|
||||
response.Execution.HTTPStatus = &status
|
||||
}
|
||||
}
|
||||
terminalMu.Unlock()
|
||||
})
|
||||
var terminalAvailable bool
|
||||
var prepared *structuredoutput.PreparedSnapshot
|
||||
var record structuredoutput.CallRecord
|
||||
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
||||
record.CallID = c.runtime.nextCallID.Add(1)
|
||||
@@ -85,40 +75,61 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
record.Started = time.Now().UTC()
|
||||
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
|
||||
onCall(record)
|
||||
finishHooks = append(finishHooks, func() {
|
||||
}
|
||||
call := &pendingCall{complete: func(result *pk.RunResult, runErr error) (structuredoutput.Response, error) {
|
||||
var response structuredoutput.Response
|
||||
mu.Lock()
|
||||
if terminalAvailable {
|
||||
response.Execution = outcomes.ExecutionCounts{
|
||||
Available: true, ProviderAttempts: int(terminal.Attempts),
|
||||
TransientRetries: int(terminal.TransientRetries), StructuralRepairs: int(terminal.RepairCalls),
|
||||
RetriesExhausted: terminal.RetriesExhausted,
|
||||
}
|
||||
if terminal.StatusCode != 0 {
|
||||
status := terminal.StatusCode
|
||||
response.Execution.HTTPStatus = &status
|
||||
}
|
||||
}
|
||||
if prepared != nil {
|
||||
response.Prepared = prepared
|
||||
response.Prompt = prepared.Prompt
|
||||
response.Model = prepared.Prompt.Model
|
||||
}
|
||||
mu.Unlock()
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
response.Model = result.ModelName
|
||||
}
|
||||
err := c.finishResponse(parent, &response, result, runErr, out)
|
||||
if onCall := c.hooks.OnCall; onCall != nil {
|
||||
if response.Prepared != nil {
|
||||
record.Prompt = response.Prompt
|
||||
}
|
||||
record.Finished = time.Now().UTC()
|
||||
if err != nil {
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
|
||||
record.Failure = structuredoutput.FailureRuntime
|
||||
if structuredoutput.IsMalformedError(err) {
|
||||
record.Failure = structuredoutput.FailureMalformedOutput
|
||||
} else {
|
||||
record.Failure = structuredoutput.FailureRuntime
|
||||
}
|
||||
}
|
||||
onCall(record)
|
||||
})
|
||||
}
|
||||
finish := func() {
|
||||
for i := len(finishHooks) - 1; i >= 0; i-- {
|
||||
finishHooks[i]()
|
||||
}
|
||||
return response, err
|
||||
}}
|
||||
fail := func(err error) (*pendingCall, error) {
|
||||
_, err = call.finish(nil, err)
|
||||
return call, err
|
||||
}
|
||||
fail := func(failure error) (*pendingCall, error) {
|
||||
err = failure
|
||||
finish()
|
||||
return nil, failure
|
||||
if err := parent.Err(); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
if parent.Err() != nil {
|
||||
return fail(classifiedFailure(parent, ctx, parent.Err(), nil))
|
||||
}
|
||||
if err = validateOutputTarget(out); err != nil {
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err})
|
||||
}
|
||||
contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts)
|
||||
if contractErr != nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr})
|
||||
contract, err := OutputContract(req.Schema, c.settings.RepairAttempts)
|
||||
if err != nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err})
|
||||
}
|
||||
vars := prompts.ProcessingVariables(req.Variables)
|
||||
var exchange *pk.Exchange
|
||||
if req.ContinueFrom != nil {
|
||||
var ok bool
|
||||
@@ -127,75 +138,68 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
|
||||
}
|
||||
}
|
||||
runRequest := pk.RunRequest{
|
||||
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
|
||||
call.request = pk.RunRequest{
|
||||
PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables),
|
||||
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
|
||||
TransientRetries: &c.settings.TransientRetries,
|
||||
Validation: contract,
|
||||
TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout,
|
||||
Validation: contract,
|
||||
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
|
||||
if event.Kind == pk.ExecutionFinished {
|
||||
terminalMu.Lock()
|
||||
mu.Lock()
|
||||
terminal = event
|
||||
terminalAvailable = true
|
||||
terminalMu.Unlock()
|
||||
mu.Unlock()
|
||||
}
|
||||
if c.hooks.OnEventWithCall != nil {
|
||||
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
||||
}
|
||||
}),
|
||||
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
|
||||
snapshot := preparedSnapshot(details)
|
||||
mu.Lock()
|
||||
prepared = snapshot
|
||||
mu.Unlock()
|
||||
},
|
||||
}
|
||||
var details pk.PreparedRun
|
||||
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
|
||||
details = prepared
|
||||
response.Prompt = promptMetadata(details)
|
||||
record.Prompt = response.Prompt
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
response.Prepared = preparedSnapshot(details)
|
||||
record.Prepared = response.Prepared
|
||||
return call, nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) finishResponse(parent context.Context, response *structuredoutput.Response, result *pk.RunResult, runErr error, out any) error {
|
||||
if runErr != nil || parent.Err() != nil {
|
||||
var typed *structuredoutput.Failure
|
||||
if parent.Err() == nil && errors.As(runErr, &typed) {
|
||||
return runErr
|
||||
}
|
||||
failure := classifiedFailure(parent, runErr, result)
|
||||
if failure.Reason == outcomes.ReasonSetupFailure && errors.Is(failure, pk.ErrAPIKeyEnvMissing) {
|
||||
failure.Message = fmt.Sprintf("structured model setup failed for %s profile %q: %v", c.role, c.profileID, failure.Cause)
|
||||
}
|
||||
return failure
|
||||
}
|
||||
runRequest.OperationTimeout = c.settings.OperationTimeout
|
||||
runRequest.OnPrepared = capturePrepared
|
||||
return &pendingCall{request: runRequest, finish: func(result *pk.RunResult, runErr error) (completed structuredoutput.Response, finalErr error) {
|
||||
defer func() { err = finalErr; finish(); completed = response }()
|
||||
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
record.FinalContent = result.RawOutput
|
||||
response.Model = result.ModelName
|
||||
response.PromptTokens = result.Usage.PromptTokens
|
||||
response.CompletionTokens = result.Usage.CompletionTokens
|
||||
response.TotalTokens = result.Usage.TotalTokens
|
||||
if result == nil {
|
||||
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
return classifiedFailure(parent, nil, result)
|
||||
}
|
||||
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
||||
var typeError *json.UnmarshalTypeError
|
||||
if errors.As(decodeErr, &typeError) {
|
||||
return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
|
||||
}
|
||||
if runErr != nil {
|
||||
return response, classifiedFailure(parent, ctx, runErr, result)
|
||||
}
|
||||
if result == nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
return response, classifiedFailure(parent, ctx, nil, result)
|
||||
}
|
||||
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
||||
var typeError *json.UnmarshalTypeError
|
||||
if errors.As(decodeErr, &typeError) {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
|
||||
}
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
||||
}
|
||||
if result.Exchange != nil {
|
||||
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
|
||||
}
|
||||
|
||||
response.Content = json.RawMessage(result.RawOutput)
|
||||
response.Provider = "openai-compatible"
|
||||
return response, nil
|
||||
}}, nil
|
||||
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
||||
}
|
||||
if result.Exchange != nil {
|
||||
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
|
||||
}
|
||||
response.Content = json.RawMessage(result.RawOutput)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent)
|
||||
if err != nil {
|
||||
return structuredoutput.Response{}, err
|
||||
return call.response, err
|
||||
}
|
||||
result, err := c.runtime.Engine.Run(ctx, call.request)
|
||||
return call.finish(result, err)
|
||||
@@ -212,13 +216,15 @@ func OutputContract(schema responseschema.Key, repairs int) (*pk.OutputContract,
|
||||
|
||||
// classifiedFailure deliberately checks setup defects before broad PromptKit
|
||||
// validation/generation wrappers. Provider text is kept only in the typed cause.
|
||||
func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
|
||||
func classifiedFailure(parent context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
|
||||
failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause}
|
||||
if parent.Err() != nil {
|
||||
failure.Class = structuredoutput.FailureCanceled
|
||||
failure.Reason = outcomes.ReasonCanceled
|
||||
failure.Message = "processing canceled"
|
||||
failure.Cause = parent.Err()
|
||||
if !errors.Is(cause, parent.Err()) {
|
||||
failure.Cause = errors.Join(parent.Err(), cause)
|
||||
}
|
||||
return failure
|
||||
}
|
||||
if errors.Is(cause, pk.ErrOperationTimeout) {
|
||||
@@ -232,7 +238,13 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
|
||||
failure.Message = "structured model setup failed"
|
||||
return failure
|
||||
}
|
||||
if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) {
|
||||
if errors.Is(cause, pk.ErrTransport) {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Reason = outcomes.ReasonTransportFailed
|
||||
failure.Message = "model transport failed"
|
||||
return failure
|
||||
}
|
||||
if errors.Is(cause, context.DeadlineExceeded) {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Reason = outcomes.ReasonOperationTimeout
|
||||
failure.Message = "model operation timed out"
|
||||
@@ -244,12 +256,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
|
||||
failure.Message = "model capacity was unavailable"
|
||||
return failure
|
||||
}
|
||||
if errors.Is(cause, pk.ErrTransport) {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Reason = outcomes.ReasonTransportFailed
|
||||
failure.Message = "model transport failed"
|
||||
return failure
|
||||
}
|
||||
if errors.Is(cause, pk.ErrMalformedResponse) {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Reason = outcomes.ReasonProviderResponseInvalid
|
||||
@@ -265,7 +271,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
|
||||
}
|
||||
if result != nil && !result.Validation.IsValid {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Kind = structuredoutput.FailureMalformedOutput
|
||||
failure.Reason = outcomes.ReasonInvalidOutputExhausted
|
||||
failure.Message = "structured output remained invalid"
|
||||
return failure
|
||||
|
||||
@@ -38,7 +38,7 @@ func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs in
|
||||
var count atomic.Int32
|
||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||
dir := t.TempDir()
|
||||
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
|
||||
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n api_key_required: true\n api_key_env: ADAPTER_KEY\n model: full-model\n - id: short-attempt\n backend: offline\n model: full-model\n timeout_seconds: 1\n"
|
||||
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -170,13 +170,13 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||
if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role {
|
||||
t.Fatalf("logical records: %+v", records)
|
||||
}
|
||||
if tc.malformed && records[1].FinalContent == "" {
|
||||
if tc.malformed && (response.FinalContent == "" || len(response.Content) != 0) {
|
||||
t.Fatal("invalid final content lost")
|
||||
}
|
||||
if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries {
|
||||
t.Fatalf("execution counts: %+v", events)
|
||||
}
|
||||
if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") {
|
||||
if err == nil && (out == nil || string(response.Content) != valid || response.Model != "full-model") {
|
||||
t.Fatalf("decoded response: %+v, %v", response, out)
|
||||
}
|
||||
})
|
||||
@@ -404,7 +404,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
|
||||
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
|
||||
blocking := func(r *http.Request) { <-r.Context().Done() }
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
|
||||
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
@@ -414,7 +414,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
|
||||
var output map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available {
|
||||
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(err, pk.ErrOperationTimeout) || !response.Execution.Available {
|
||||
t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution)
|
||||
}
|
||||
|
||||
@@ -426,6 +426,58 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterDistinguishesAttemptTimeoutFromParentDeadline(t *testing.T) {
|
||||
blocking := func(r *http.Request) { <-r.Context().Done() }
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
|
||||
client, err := runtime.NewRoleClient("proposal", "short-attempt", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
parentBudget time.Duration
|
||||
class outcomes.FailureClass
|
||||
reason outcomes.ReasonCode
|
||||
}{
|
||||
{"attempt", 5 * time.Second, outcomes.FailureRecoverable, outcomes.ReasonTransportFailed},
|
||||
{"parent", 300 * time.Millisecond, outcomes.FailureCanceled, outcomes.ReasonCanceled},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
parent, cancel := context.WithTimeout(context.Background(), tc.parentBudget)
|
||||
defer cancel()
|
||||
before := count.Load()
|
||||
var output map[string]any
|
||||
response, err := client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output)
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != tc.class || failure.Reason != tc.reason ||
|
||||
!errors.Is(err, context.DeadlineExceeded) || errors.Is(err, pk.ErrOperationTimeout) ||
|
||||
!response.Execution.Available || count.Load() != before+1 {
|
||||
t.Fatalf("deadline classification: failure=%#v error=%v counts=%+v requests=%d", failure, err, response.Execution, count.Load()-before)
|
||||
}
|
||||
if tc.class == outcomes.FailureRecoverable && (!errors.Is(err, pk.ErrTransport) || parent.Err() != nil) {
|
||||
t.Fatalf("missing transport cause or parent also expired: error=%v parent=%v", err, parent.Err())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterMissingCredentialRetainsSetupCause(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, nil, 0, 0, nil)
|
||||
t.Setenv("ADAPTER_KEY", "")
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output map[string]any
|
||||
_, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != outcomes.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure ||
|
||||
!errors.Is(err, pk.ErrAPIKeyEnvMissing) || !strings.Contains(err.Error(), "ADAPTER_KEY") ||
|
||||
!strings.Contains(err.Error(), `proposal profile "base"`) || count.Load() != 0 {
|
||||
t.Fatalf("missing credential classification: failure=%#v error=%v requests=%d", failure, err, count.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"reason":"ok"}]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
|
||||
@@ -477,7 +529,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
|
||||
parent, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation)
|
||||
failure := classifiedFailure(parent, parent, cause, nil)
|
||||
failure := classifiedFailure(parent, cause, nil)
|
||||
if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) {
|
||||
t.Fatalf("unknown validation failure: %#v", failure)
|
||||
}
|
||||
@@ -488,7 +540,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
|
||||
func TestOperationTimeoutDuringPreparationIsRecoverable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cause := errors.Join(pk.ErrPromptLoad, pk.ErrOperationTimeout, context.DeadlineExceeded)
|
||||
failure := classifiedFailure(ctx, ctx, cause, nil)
|
||||
failure := classifiedFailure(ctx, cause, nil)
|
||||
if failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(failure, pk.ErrOperationTimeout) {
|
||||
t.Fatalf("failure = %#v", failure)
|
||||
}
|
||||
|
||||
@@ -36,14 +36,15 @@ func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exc
|
||||
return &pk.RunRequest{}, nil
|
||||
}
|
||||
var output json.RawMessage
|
||||
pending, err = c.prepareCall(ctx, *selected, &output, nil)
|
||||
call, err := c.prepareCall(ctx, *selected, &output, nil)
|
||||
if err != nil {
|
||||
_, domainErr := next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: err})
|
||||
_, domainErr := next(ctx, structuredoutput.Turn{Response: &call.response, Err: err})
|
||||
if domainErr != nil {
|
||||
return nil, domainErr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
pending = call
|
||||
return &pending.request, nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -41,8 +41,16 @@ type ExecutionCounts struct {
|
||||
HTTPStatus *int `json:"http_status,omitempty"`
|
||||
}
|
||||
|
||||
// ReasonCode is a fixed, safe explanation category. Raw provider errors and
|
||||
// prompt content are never stored in an operation outcome.
|
||||
// FailureClass determines whether processing can skip an operation or must stop.
|
||||
type FailureClass string
|
||||
|
||||
const (
|
||||
FailureRecoverable FailureClass = "recoverable"
|
||||
FailureFatal FailureClass = "fatal"
|
||||
FailureCanceled FailureClass = "canceled"
|
||||
)
|
||||
|
||||
// ReasonCode is a fixed explanation category for the serialized outcome.
|
||||
type ReasonCode string
|
||||
|
||||
const (
|
||||
@@ -85,6 +93,8 @@ type Operation struct {
|
||||
Reference Reference `json:"reference"`
|
||||
Counts ExecutionCounts `json:"counts"`
|
||||
Succeeded bool `json:"succeeded"`
|
||||
FailureClass FailureClass `json:"-"`
|
||||
Cause error `json:"-"`
|
||||
FailureReason ReasonCode `json:"failure_reason,omitempty"`
|
||||
RecoveryReasons []RecoveryReason `json:"recovery_reasons,omitempty"`
|
||||
ProposalIndices []int `json:"proposal_indices,omitempty"`
|
||||
|
||||
@@ -5,7 +5,6 @@ package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@@ -129,15 +128,9 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(callErr, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
}
|
||||
if operation.FailureReason == "" {
|
||||
operation.FailureReason = outcomes.ReasonInternalFailure
|
||||
}
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
|
||||
if structuredoutput.IsRecoverable(callErr) && failure != nil && failure.Reason != "" {
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
return result, nil
|
||||
}
|
||||
return result, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
|
||||
@@ -226,6 +226,20 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesUsesInternalReasonForUnspecifiedRecoverableFailure(t *testing.T) {
|
||||
cause := errors.New("underlying call failure")
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = &fakeStructuredClient{err: &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Cause: cause}}
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil || len(result.Operations) != 1 {
|
||||
t.Fatalf("recoverable proposal returned operations=%+v error=%v", result.Operations, err)
|
||||
}
|
||||
operation := result.Operations[0]
|
||||
if operation.FailureClass != outcomes.FailureRecoverable || operation.FailureReason != outcomes.ReasonInternalFailure || !errors.Is(operation.Cause, cause) {
|
||||
t.Fatalf("operation lost typed failure facts: %+v", operation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}}
|
||||
req := defaultRequest(t)
|
||||
|
||||
@@ -2,7 +2,6 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
@@ -174,14 +173,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
branch.Attempts = append(branch.Attempts, record)
|
||||
}
|
||||
if callErr != nil {
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(callErr, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
} else {
|
||||
operation.FailureReason = outcomes.ReasonInternalFailure
|
||||
}
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
|
||||
if structuredoutput.IsRecoverable(callErr) {
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
}
|
||||
@@ -190,7 +184,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
var err error
|
||||
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
operation.FailureReason = outcomes.ReasonInvalidOutput
|
||||
structuredoutput.RecordFailure(&operation, err)
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "unavailable"
|
||||
return nil, nil
|
||||
|
||||
@@ -189,7 +189,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
{"exhausted", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`}, nil, 2, "exhausted", 2},
|
||||
{"no-effect revision", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}}`}, nil, 2, "rejected", 1},
|
||||
{"wrong reference", []string{`{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "unavailable", 1},
|
||||
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
|
||||
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed", Cause: errors.New("underlying provider error")}, 1, "unavailable", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
@@ -199,13 +199,20 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
result, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
|
||||
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
|
||||
}
|
||||
if tc.err != nil {
|
||||
operation := result.operations[len(result.operations)-1]
|
||||
if operation.FailureClass != outcomes.FailureRecoverable ||
|
||||
operation.FailureReason != outcomes.ReasonProviderFailed || !errors.Is(operation.Cause, tc.err) {
|
||||
t.Fatalf("repair lost typed failure facts: %+v", operation)
|
||||
}
|
||||
}
|
||||
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
|
||||
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
|
||||
}
|
||||
|
||||
@@ -50,17 +50,13 @@ type Exchange struct {
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Exchange *Exchange `json:"-"`
|
||||
Execution outcomes.ExecutionCounts `json:"execution"`
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
Exchange *Exchange `json:"-"`
|
||||
Execution outcomes.ExecutionCounts `json:"execution"`
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
}
|
||||
|
||||
// PreparedSnapshot is the exact rendered request snapshot. Its messages may
|
||||
@@ -81,8 +77,7 @@ type CacheControl struct {
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// CallRecord brackets one logical operation, including preparation. Content
|
||||
// is present only when a model returned it; consumers must protect it.
|
||||
// CallRecord brackets one logical operation, including preparation.
|
||||
type CallRecord struct {
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
@@ -90,8 +85,6 @@ type CallRecord struct {
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
Failure FailureKind `json:"failure,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
@@ -14,10 +14,8 @@ const (
|
||||
FailureRuntime FailureKind = "runtime"
|
||||
)
|
||||
|
||||
// Failure exposes a safe message while retaining the typed cause for callers.
|
||||
// The cause must not be serialized or formatted directly in user artifacts.
|
||||
// Failure exposes an operator-facing message and retains the typed cause.
|
||||
type Failure struct {
|
||||
Kind FailureKind // Retained until the adapter switches to reason-based classification.
|
||||
Class FailureClass
|
||||
Reason outcomes.ReasonCode
|
||||
Message string
|
||||
@@ -26,12 +24,12 @@ type Failure struct {
|
||||
|
||||
// FailureClass determines whether domain processing may safely skip a model
|
||||
// operation or must stop the run. Parent cancellation always stops the run.
|
||||
type FailureClass string
|
||||
type FailureClass = outcomes.FailureClass
|
||||
|
||||
const (
|
||||
FailureRecoverable FailureClass = "recoverable"
|
||||
FailureFatal FailureClass = "fatal"
|
||||
FailureCanceled FailureClass = "canceled"
|
||||
FailureRecoverable = outcomes.FailureRecoverable
|
||||
FailureFatal = outcomes.FailureFatal
|
||||
FailureCanceled = outcomes.FailureCanceled
|
||||
)
|
||||
|
||||
func (f *Failure) Error() string { return f.Message }
|
||||
@@ -42,7 +40,22 @@ func IsRecoverable(err error) bool {
|
||||
return errors.As(err, &failure) && failure.Class == FailureRecoverable
|
||||
}
|
||||
|
||||
// RecordFailure retains the error chain for processing and sets the compact
|
||||
// reason used by reports. An absent reason does not change recoverability.
|
||||
func RecordFailure(operation *outcomes.Operation, err error) {
|
||||
operation.Cause = err
|
||||
operation.FailureClass = FailureFatal
|
||||
operation.FailureReason = outcomes.ReasonInternalFailure
|
||||
var failure *Failure
|
||||
if errors.As(err, &failure) {
|
||||
operation.FailureClass = failure.Class
|
||||
if failure.Reason != "" {
|
||||
operation.FailureReason = failure.Reason
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IsMalformedError(err error) bool {
|
||||
var failure *Failure
|
||||
return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput
|
||||
return errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted
|
||||
}
|
||||
|
||||
@@ -9,11 +9,11 @@ import (
|
||||
|
||||
func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) {
|
||||
cause := errors.New("provider token secret-value")
|
||||
malformed := &Failure{Kind: FailureMalformedOutput, Message: "malformed structured output", Cause: cause}
|
||||
malformed := &Failure{Class: FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "malformed structured output", Cause: cause}
|
||||
if !IsMalformedError(malformed) || !errors.Is(malformed, cause) || malformed.Error() != "malformed structured output" {
|
||||
t.Fatalf("malformed classification: %v", malformed)
|
||||
}
|
||||
runtime := &Failure{Kind: FailureRuntime, Message: "structured model call failed", Cause: cause}
|
||||
runtime := &Failure{Class: FailureFatal, Message: "structured model call failed", Cause: cause}
|
||||
if IsMalformedError(runtime) || IsMalformedError(cause) || IsMalformedError(nil) || !errors.Is(runtime, cause) || runtime.Error() != "structured model call failed" {
|
||||
t.Fatalf("runtime classification: %v", runtime)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
cause := errors.New("underlying transport error")
|
||||
failed := &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed", Cause: cause}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
@@ -67,6 +68,10 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
|
||||
t.Fatalf("batch operations: %+v", result.Operations)
|
||||
}
|
||||
operation := result.Operations[1]
|
||||
if operation.FailureClass != outcomes.FailureRecoverable || !errors.Is(operation.Cause, cause) {
|
||||
t.Fatalf("validation lost typed failure facts: %+v", operation)
|
||||
}
|
||||
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package validators
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -103,6 +102,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
operations = append(operations, outcomes.Operation{
|
||||
Reference: operationRef(outcomes.OperationValidationInput, nil),
|
||||
FailureClass: outcomes.FailureRecoverable,
|
||||
FailureReason: outcomes.ReasonValidationInputTooLarge,
|
||||
ProposalIndices: indices,
|
||||
})
|
||||
@@ -173,17 +173,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
if err != nil {
|
||||
artifacts := writeInteraction(nil, err)
|
||||
var failure *structuredoutput.Failure
|
||||
if errors.As(err, &failure) {
|
||||
operation.FailureReason = failure.Reason
|
||||
}
|
||||
if operation.FailureReason == "" {
|
||||
operation.FailureReason = outcomes.ReasonInternalFailure
|
||||
}
|
||||
structuredoutput.RecordFailure(&operation, err)
|
||||
operation.ProposalIndices = batchProposalIndices(batch.Items)
|
||||
operations = append(operations, operation)
|
||||
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
|
||||
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable")
|
||||
for i := range decisions {
|
||||
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
|
||||
}
|
||||
@@ -196,6 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
reconciled := reconcileLLMResponse(batch.Items, response)
|
||||
artifacts := writeInteraction(&reconciled, nil)
|
||||
if len(reconciled.UnavailableIndices) > 0 {
|
||||
operation.FailureClass = outcomes.FailureRecoverable
|
||||
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
|
||||
operation.ProposalIndices = reconciled.UnavailableIndices
|
||||
} else {
|
||||
|
||||
@@ -191,7 +191,7 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
||||
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = client
|
||||
|
||||
Reference in New Issue
Block a user