Unify structured call completion and retain typed failures

This commit is contained in:
2026-11-16 14:27:54 +00:00
parent e98e525b5c
commit c2c8050bc8
18 changed files with 313 additions and 186 deletions

View File

@@ -1206,6 +1206,30 @@ func TestSubprocessInfeasibleSectionTargetIsAnInputFailure(t *testing.T) {
}
}
func TestSubprocessMissingCredentialIdentifiesRequiredEnvironmentVariable(t *testing.T) {
f := newProcessFixture(t, "success", "[grammar]")
f.provider.secret = ""
resources := filepath.Join(f.dir, "prompts.yml")
body, err := os.ReadFile(resources)
if err != nil {
t.Fatal(err)
}
body = bytes.Replace(body, []byte(" - id: proposal\n"), []byte(" - id: proposal\n api_key_required: true\n api_key_env: AUDITA_TEST_PROVIDER_KEY\n"), 1)
if err := os.WriteFile(resources, body, 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 1 || stdout != "" || f.provider.callCount() != 0 ||
!strings.Contains(stderr, "AUDITA_TEST_PROVIDER_KEY") || !strings.Contains(stderr, `proposal profile "proposal"`) {
t.Fatalf("missing credential: status=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
document := f.errors(t)
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 ||
document.Errors[0].Reason != "setup_failure" || strings.Contains(document.Errors[0].Message, "AUDITA_TEST_PROVIDER_KEY") {
t.Fatalf("expected compact setup failure in ordinary report: %+v", document)
}
}
func TestSubprocessUnknownProfileFailsBeforeModelDispatchAndOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true, "--proposal-profile", "unknown")

View File

@@ -12,10 +12,14 @@ profiles and effective budgets. Clients share engine-local backend admission.
The adapter uses ordinary `Run` with `OperationTimeout` and `OnPrepared`, passes
Audita's per-call schema and repair budget, and decodes only validated results.
The operation budget includes preparation, admission, retries, and structural
repair. `ErrOperationTimeout` remains recoverable even when wrapped by a
preparation error. Parent cancellation remains distinct. The prepared notification
repair. An expired parent context stops processing and retains any native cause.
With an active parent, PromptKit's operation deadline remains recoverable even
when a preparation error wraps it. A native transport timeout affects the HTTP
attempt and is recorded as a transport failure. The prepared notification
provides the actual request snapshot; failures before notification have no snapshot.
PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md)
PromptKit owns backend and credential selection. When a required credential is absent,
the CLI identifies the role, profile, and environment variable using the native
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)
for failure behavior and the [configuration reference](../config.md) for sources.
When correction repair is enabled, the proposal call captures a native immutable
@@ -26,7 +30,10 @@ for assessment. Follow-ups select the repair prompt, profile, and output schema
directly; Audita does not rerender the original proposal or assemble replay messages.
An opaque exchange in the structured-output contract keeps provider state outside
modules and validators. The adapter shares request construction, notifications,
observers, and result/failure finalization between ordinary and coordinated calls.
observers, and once-only completion between ordinary and coordinated calls.
Completion attaches terminal counts when available and records one logical finish.
Prepared requests and terminal output travel in the returned response for an
optional interaction artifact. Logical call notifications carry metadata.
Audita owns actionable rejection, revision decoding, withdrawal, current previews,
and full revalidation. Accepted siblings remain independent. Requesting more work
@@ -44,7 +51,11 @@ selected runs in a native `ExecutionGroup`. Proposal, repair, and nested validat
clients explicitly share the invocation observer; history reuse adds no extra run.
Native summaries deduplicate terminal totals by run ID and retain terminal coverage
separately from usage presence and completeness. Logical calls and run membership
can differ, especially on failures before execution. Ordinary
reports use compact safe outcomes; request and response evidence is optional
debug material. The runner preserves ordered pipeline traversal and deterministic
section collection while canceling and joining peers on fatal errors.
can differ, especially on failures before execution.
Processing operations retain the failure class, reason, and underlying error.
When a client omits the reason, the operation uses `internal_failure` and keeps
the original class and error chain. Ordinary reports use compact outcomes.
Request and response evidence is optional debug material. The runner preserves
ordered pipeline traversal and deterministic section collection while canceling
and joining peers on fatal errors.

View File

@@ -447,6 +447,14 @@ controlled budgets. Check setup class/cause and useful native information,
plus ordinary/coordinator terminal and undelivered-evidence paths. Update
[LLM runtime internals](../internal/llm-runtime.md).
**Complete.** Ordinary and coordinated calls share once-only completion,
including prepared and undelivered evidence and native terminal counts.
Proposal, validation, and repair retain typed failure facts. Independent
parent, operation, and attempt deadlines produced the expected classifications.
The CLI identifies a missing required credential by role, profile, and
environment variable while the ordinary report retains the setup reason.
The CLI build, full offline Go suite, and targeted race checks passed.
## Stage 13 — Use one proposal handoff
Audit #46.

View File

@@ -68,7 +68,9 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
var cause *structuredoutput.Failure
if errors.As(runErr, &cause) {
failure.Reason = cause.Reason
if cause.Reason != "" {
failure.Reason = cause.Reason
}
} else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) {
failure.Reason = outcomes.ReasonCanceled
}

View File

@@ -17,7 +17,7 @@ import (
)
// CallHooks receive one logical start and finish record and the execution
// events for this call. Call records can contain transcript content.
// events associated with the call.
type CallHooks struct {
OnCall func(structuredoutput.CallRecord)
OnEvent pk.ExecutionObserver
@@ -45,35 +45,25 @@ func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hoo
}
type pendingCall struct {
request pk.RunRequest
finish func(*pk.RunResult, error) (structuredoutput.Response, error)
request pk.RunRequest
once sync.Once
complete func(*pk.RunResult, error) (structuredoutput.Response, error)
response structuredoutput.Response
err error
}
func (p *pendingCall) finish(result *pk.RunResult, err error) (structuredoutput.Response, error) {
p.once.Do(func() {
p.response, p.err = p.complete(result, err)
})
return p.response, p.err
}
func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) {
var response structuredoutput.Response
var err error
var finishHooks []func()
ctx := parent
var terminalMu sync.Mutex
var mu sync.Mutex
var terminal pk.ExecutionEvent
terminalAvailable := false
finishHooks = append(finishHooks, func() {
terminalMu.Lock()
if terminalAvailable {
response.Execution = outcomes.ExecutionCounts{
Available: true,
ProviderAttempts: int(terminal.Attempts),
TransientRetries: int(terminal.TransientRetries),
StructuralRepairs: int(terminal.RepairCalls),
RetriesExhausted: terminal.RetriesExhausted,
}
if terminal.StatusCode != 0 {
status := terminal.StatusCode
response.Execution.HTTPStatus = &status
}
}
terminalMu.Unlock()
})
var terminalAvailable bool
var prepared *structuredoutput.PreparedSnapshot
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
@@ -85,40 +75,61 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
record.Started = time.Now().UTC()
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
onCall(record)
finishHooks = append(finishHooks, func() {
}
call := &pendingCall{complete: func(result *pk.RunResult, runErr error) (structuredoutput.Response, error) {
var response structuredoutput.Response
mu.Lock()
if terminalAvailable {
response.Execution = outcomes.ExecutionCounts{
Available: true, ProviderAttempts: int(terminal.Attempts),
TransientRetries: int(terminal.TransientRetries), StructuralRepairs: int(terminal.RepairCalls),
RetriesExhausted: terminal.RetriesExhausted,
}
if terminal.StatusCode != 0 {
status := terminal.StatusCode
response.Execution.HTTPStatus = &status
}
}
if prepared != nil {
response.Prepared = prepared
response.Prompt = prepared.Prompt
response.Model = prepared.Prompt.Model
}
mu.Unlock()
if result != nil {
response.FinalContent = result.RawOutput
response.Model = result.ModelName
}
err := c.finishResponse(parent, &response, result, runErr, out)
if onCall := c.hooks.OnCall; onCall != nil {
if response.Prepared != nil {
record.Prompt = response.Prompt
}
record.Finished = time.Now().UTC()
if err != nil {
var failure *structuredoutput.Failure
if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted {
record.Failure = structuredoutput.FailureRuntime
if structuredoutput.IsMalformedError(err) {
record.Failure = structuredoutput.FailureMalformedOutput
} else {
record.Failure = structuredoutput.FailureRuntime
}
}
onCall(record)
})
}
finish := func() {
for i := len(finishHooks) - 1; i >= 0; i-- {
finishHooks[i]()
}
return response, err
}}
fail := func(err error) (*pendingCall, error) {
_, err = call.finish(nil, err)
return call, err
}
fail := func(failure error) (*pendingCall, error) {
err = failure
finish()
return nil, failure
if err := parent.Err(); err != nil {
return fail(err)
}
if parent.Err() != nil {
return fail(classifiedFailure(parent, ctx, parent.Err(), nil))
}
if err = validateOutputTarget(out); err != nil {
if err := validateOutputTarget(out); err != nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err})
}
contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts)
if contractErr != nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr})
contract, err := OutputContract(req.Schema, c.settings.RepairAttempts)
if err != nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err})
}
vars := prompts.ProcessingVariables(req.Variables)
var exchange *pk.Exchange
if req.ContinueFrom != nil {
var ok bool
@@ -127,75 +138,68 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
}
}
runRequest := pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
call.request = pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables),
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
TransientRetries: &c.settings.TransientRetries,
Validation: contract,
TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout,
Validation: contract,
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
if event.Kind == pk.ExecutionFinished {
terminalMu.Lock()
mu.Lock()
terminal = event
terminalAvailable = true
terminalMu.Unlock()
mu.Unlock()
}
if c.hooks.OnEventWithCall != nil {
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
}
}),
OnPrepared: func(_ context.Context, details pk.PreparedRun) {
snapshot := preparedSnapshot(details)
mu.Lock()
prepared = snapshot
mu.Unlock()
},
}
var details pk.PreparedRun
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
details = prepared
response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model
response.Prepared = preparedSnapshot(details)
record.Prepared = response.Prepared
return call, nil
}
func (c *RoleClient) finishResponse(parent context.Context, response *structuredoutput.Response, result *pk.RunResult, runErr error, out any) error {
if runErr != nil || parent.Err() != nil {
var typed *structuredoutput.Failure
if parent.Err() == nil && errors.As(runErr, &typed) {
return runErr
}
failure := classifiedFailure(parent, runErr, result)
if failure.Reason == outcomes.ReasonSetupFailure && errors.Is(failure, pk.ErrAPIKeyEnvMissing) {
failure.Message = fmt.Sprintf("structured model setup failed for %s profile %q: %v", c.role, c.profileID, failure.Cause)
}
return failure
}
runRequest.OperationTimeout = c.settings.OperationTimeout
runRequest.OnPrepared = capturePrepared
return &pendingCall{request: runRequest, finish: func(result *pk.RunResult, runErr error) (completed structuredoutput.Response, finalErr error) {
defer func() { err = finalErr; finish(); completed = response }()
if result != nil {
response.FinalContent = result.RawOutput
record.FinalContent = result.RawOutput
response.Model = result.ModelName
response.PromptTokens = result.Usage.PromptTokens
response.CompletionTokens = result.Usage.CompletionTokens
response.TotalTokens = result.Usage.TotalTokens
if result == nil {
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
}
if !result.Validation.IsValid {
return classifiedFailure(parent, nil, result)
}
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
var typeError *json.UnmarshalTypeError
if errors.As(decodeErr, &typeError) {
return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
}
if runErr != nil {
return response, classifiedFailure(parent, ctx, runErr, result)
}
if result == nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
}
if !result.Validation.IsValid {
return response, classifiedFailure(parent, ctx, nil, result)
}
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
var typeError *json.UnmarshalTypeError
if errors.As(decodeErr, &typeError) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
if result.Exchange != nil {
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
}
response.Content = json.RawMessage(result.RawOutput)
response.Provider = "openai-compatible"
return response, nil
}}, nil
return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
if result.Exchange != nil {
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
}
response.Content = json.RawMessage(result.RawOutput)
return nil
}
func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent)
if err != nil {
return structuredoutput.Response{}, err
return call.response, err
}
result, err := c.runtime.Engine.Run(ctx, call.request)
return call.finish(result, err)
@@ -212,13 +216,15 @@ func OutputContract(schema responseschema.Key, repairs int) (*pk.OutputContract,
// classifiedFailure deliberately checks setup defects before broad PromptKit
// validation/generation wrappers. Provider text is kept only in the typed cause.
func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
func classifiedFailure(parent context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure {
failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause}
if parent.Err() != nil {
failure.Class = structuredoutput.FailureCanceled
failure.Reason = outcomes.ReasonCanceled
failure.Message = "processing canceled"
failure.Cause = parent.Err()
if !errors.Is(cause, parent.Err()) {
failure.Cause = errors.Join(parent.Err(), cause)
}
return failure
}
if errors.Is(cause, pk.ErrOperationTimeout) {
@@ -232,7 +238,13 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
failure.Message = "structured model setup failed"
return failure
}
if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) {
if errors.Is(cause, pk.ErrTransport) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonTransportFailed
failure.Message = "model transport failed"
return failure
}
if errors.Is(cause, context.DeadlineExceeded) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonOperationTimeout
failure.Message = "model operation timed out"
@@ -244,12 +256,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
failure.Message = "model capacity was unavailable"
return failure
}
if errors.Is(cause, pk.ErrTransport) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonTransportFailed
failure.Message = "model transport failed"
return failure
}
if errors.Is(cause, pk.ErrMalformedResponse) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonProviderResponseInvalid
@@ -265,7 +271,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
}
if result != nil && !result.Validation.IsValid {
failure.Class = structuredoutput.FailureRecoverable
failure.Kind = structuredoutput.FailureMalformedOutput
failure.Reason = outcomes.ReasonInvalidOutputExhausted
failure.Message = "structured output remained invalid"
return failure

View File

@@ -38,7 +38,7 @@ func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs in
var count atomic.Int32
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n api_key_required: true\n api_key_env: ADAPTER_KEY\n model: full-model\n - id: short-attempt\n backend: offline\n model: full-model\n timeout_seconds: 1\n"
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
@@ -170,13 +170,13 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role {
t.Fatalf("logical records: %+v", records)
}
if tc.malformed && records[1].FinalContent == "" {
if tc.malformed && (response.FinalContent == "" || len(response.Content) != 0) {
t.Fatal("invalid final content lost")
}
if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries {
t.Fatalf("execution counts: %+v", events)
}
if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") {
if err == nil && (out == nil || string(response.Content) != valid || response.Model != "full-model") {
t.Fatalf("decoded response: %+v, %v", response, out)
}
})
@@ -404,7 +404,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
}
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
blocking := func(r *http.Request) { <-r.Context().Done() }
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
@@ -414,7 +414,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
var output map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available {
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(err, pk.ErrOperationTimeout) || !response.Execution.Available {
t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution)
}
@@ -426,6 +426,58 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
}
}
func TestPreparedAdapterDistinguishesAttemptTimeoutFromParentDeadline(t *testing.T) {
blocking := func(r *http.Request) { <-r.Context().Done() }
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
client, err := runtime.NewRoleClient("proposal", "short-attempt", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
parentBudget time.Duration
class outcomes.FailureClass
reason outcomes.ReasonCode
}{
{"attempt", 5 * time.Second, outcomes.FailureRecoverable, outcomes.ReasonTransportFailed},
{"parent", 300 * time.Millisecond, outcomes.FailureCanceled, outcomes.ReasonCanceled},
} {
t.Run(tc.name, func(t *testing.T) {
parent, cancel := context.WithTimeout(context.Background(), tc.parentBudget)
defer cancel()
before := count.Load()
var output map[string]any
response, err := client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != tc.class || failure.Reason != tc.reason ||
!errors.Is(err, context.DeadlineExceeded) || errors.Is(err, pk.ErrOperationTimeout) ||
!response.Execution.Available || count.Load() != before+1 {
t.Fatalf("deadline classification: failure=%#v error=%v counts=%+v requests=%d", failure, err, response.Execution, count.Load()-before)
}
if tc.class == outcomes.FailureRecoverable && (!errors.Is(err, pk.ErrTransport) || parent.Err() != nil) {
t.Fatalf("missing transport cause or parent also expired: error=%v parent=%v", err, parent.Err())
}
})
}
}
func TestPreparedAdapterMissingCredentialRetainsSetupCause(t *testing.T) {
runtime, count, _ := adapterFixture(t, nil, 0, 0, nil)
t.Setenv("ADAPTER_KEY", "")
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
_, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output)
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != outcomes.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure ||
!errors.Is(err, pk.ErrAPIKeyEnvMissing) || !strings.Contains(err.Error(), "ADAPTER_KEY") ||
!strings.Contains(err.Error(), `proposal profile "base"`) || count.Load() != 0 {
t.Fatalf("missing credential classification: failure=%#v error=%v requests=%d", failure, err, count.Load())
}
}
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
@@ -477,7 +529,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
defer cancel()
cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation)
failure := classifiedFailure(parent, parent, cause, nil)
failure := classifiedFailure(parent, cause, nil)
if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) {
t.Fatalf("unknown validation failure: %#v", failure)
}
@@ -488,7 +540,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
func TestOperationTimeoutDuringPreparationIsRecoverable(t *testing.T) {
ctx := context.Background()
cause := errors.Join(pk.ErrPromptLoad, pk.ErrOperationTimeout, context.DeadlineExceeded)
failure := classifiedFailure(ctx, ctx, cause, nil)
failure := classifiedFailure(ctx, cause, nil)
if failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(failure, pk.ErrOperationTimeout) {
t.Fatalf("failure = %#v", failure)
}

View File

@@ -36,14 +36,15 @@ func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exc
return &pk.RunRequest{}, nil
}
var output json.RawMessage
pending, err = c.prepareCall(ctx, *selected, &output, nil)
call, err := c.prepareCall(ctx, *selected, &output, nil)
if err != nil {
_, domainErr := next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: err})
_, domainErr := next(ctx, structuredoutput.Turn{Response: &call.response, Err: err})
if domainErr != nil {
return nil, domainErr
}
return nil, err
}
pending = call
return &pending.request, nil
},
})

View File

@@ -41,8 +41,16 @@ type ExecutionCounts struct {
HTTPStatus *int `json:"http_status,omitempty"`
}
// ReasonCode is a fixed, safe explanation category. Raw provider errors and
// prompt content are never stored in an operation outcome.
// FailureClass determines whether processing can skip an operation or must stop.
type FailureClass string
const (
FailureRecoverable FailureClass = "recoverable"
FailureFatal FailureClass = "fatal"
FailureCanceled FailureClass = "canceled"
)
// ReasonCode is a fixed explanation category for the serialized outcome.
type ReasonCode string
const (
@@ -85,6 +93,8 @@ type Operation struct {
Reference Reference `json:"reference"`
Counts ExecutionCounts `json:"counts"`
Succeeded bool `json:"succeeded"`
FailureClass FailureClass `json:"-"`
Cause error `json:"-"`
FailureReason ReasonCode `json:"failure_reason,omitempty"`
RecoveryReasons []RecoveryReason `json:"recovery_reasons,omitempty"`
ProposalIndices []int `json:"proposal_indices,omitempty"`

View File

@@ -5,7 +5,6 @@ package proposal_generation
import (
"context"
"errors"
"fmt"
"strings"
@@ -129,15 +128,9 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
}
if callErr != nil {
var failure *structuredoutput.Failure
if errors.As(callErr, &failure) {
operation.FailureReason = failure.Reason
}
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
structuredoutput.RecordFailure(&operation, callErr)
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
if structuredoutput.IsRecoverable(callErr) && failure != nil && failure.Reason != "" {
if operation.FailureClass == outcomes.FailureRecoverable {
return result, nil
}
return result, fmt.Errorf("proposal generation completion failed: %w", callErr)

View File

@@ -226,6 +226,20 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
}
}
func TestGenerateCandidatesUsesInternalReasonForUnspecifiedRecoverableFailure(t *testing.T) {
cause := errors.New("underlying call failure")
req := defaultRequest(t)
req.LLMClient = &fakeStructuredClient{err: &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Cause: cause}}
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Operations) != 1 {
t.Fatalf("recoverable proposal returned operations=%+v error=%v", result.Operations, err)
}
operation := result.Operations[0]
if operation.FailureClass != outcomes.FailureRecoverable || operation.FailureReason != outcomes.ReasonInternalFailure || !errors.Is(operation.Cause, cause) {
t.Fatalf("operation lost typed failure facts: %+v", operation)
}
}
func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) {
client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}}
req := defaultRequest(t)

View File

@@ -2,7 +2,6 @@ package runner
import (
"context"
"errors"
"fmt"
"slices"
@@ -174,14 +173,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
branch.Attempts = append(branch.Attempts, record)
}
if callErr != nil {
var failure *structuredoutput.Failure
if errors.As(callErr, &failure) {
operation.FailureReason = failure.Reason
} else {
operation.FailureReason = outcomes.ReasonInternalFailure
}
structuredoutput.RecordFailure(&operation, callErr)
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
if structuredoutput.IsRecoverable(callErr) {
if operation.FailureClass == outcomes.FailureRecoverable {
branch.Outcome = "unavailable"
return nil, nil
}
@@ -190,7 +184,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
var err error
response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil {
operation.FailureReason = outcomes.ReasonInvalidOutput
structuredoutput.RecordFailure(&operation, err)
finishAttempt(err, nil, validateSectionCandidatesResult{})
branch.Outcome = "unavailable"
return nil, nil

View File

@@ -189,7 +189,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
{"exhausted", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed", Cause: errors.New("underlying provider error")}, 1, "unavailable", 1},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
@@ -199,13 +199,20 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
result, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
}
if tc.err != nil {
operation := result.operations[len(result.operations)-1]
if operation.FailureClass != outcomes.FailureRecoverable ||
operation.FailureReason != outcomes.ReasonProviderFailed || !errors.Is(operation.Cause, tc.err) {
t.Fatalf("repair lost typed failure facts: %+v", operation)
}
}
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
}

View File

@@ -50,17 +50,13 @@ type Exchange struct {
}
type Response struct {
Exchange *Exchange `json:"-"`
Execution outcomes.ExecutionCounts `json:"execution"`
Content json.RawMessage `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Prompt PromptMetadata `json:"prompt"`
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
FinalContent string `json:"final_content,omitempty"`
Exchange *Exchange `json:"-"`
Execution outcomes.ExecutionCounts `json:"execution"`
Content json.RawMessage `json:"content,omitempty"`
Model string `json:"model,omitempty"`
Prompt PromptMetadata `json:"prompt"`
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
FinalContent string `json:"final_content,omitempty"`
}
// PreparedSnapshot is the exact rendered request snapshot. Its messages may
@@ -81,8 +77,7 @@ type CacheControl struct {
TTL string `json:"ttl,omitempty"`
}
// CallRecord brackets one logical operation, including preparation. Content
// is present only when a model returned it; consumers must protect it.
// CallRecord brackets one logical operation, including preparation.
type CallRecord struct {
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
@@ -90,8 +85,6 @@ type CallRecord struct {
Prompt PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
FinalContent string `json:"final_content,omitempty"`
Failure FailureKind `json:"failure,omitempty"`
}

View File

@@ -14,10 +14,8 @@ const (
FailureRuntime FailureKind = "runtime"
)
// Failure exposes a safe message while retaining the typed cause for callers.
// The cause must not be serialized or formatted directly in user artifacts.
// Failure exposes an operator-facing message and retains the typed cause.
type Failure struct {
Kind FailureKind // Retained until the adapter switches to reason-based classification.
Class FailureClass
Reason outcomes.ReasonCode
Message string
@@ -26,12 +24,12 @@ type Failure struct {
// FailureClass determines whether domain processing may safely skip a model
// operation or must stop the run. Parent cancellation always stops the run.
type FailureClass string
type FailureClass = outcomes.FailureClass
const (
FailureRecoverable FailureClass = "recoverable"
FailureFatal FailureClass = "fatal"
FailureCanceled FailureClass = "canceled"
FailureRecoverable = outcomes.FailureRecoverable
FailureFatal = outcomes.FailureFatal
FailureCanceled = outcomes.FailureCanceled
)
func (f *Failure) Error() string { return f.Message }
@@ -42,7 +40,22 @@ func IsRecoverable(err error) bool {
return errors.As(err, &failure) && failure.Class == FailureRecoverable
}
// RecordFailure retains the error chain for processing and sets the compact
// reason used by reports. An absent reason does not change recoverability.
func RecordFailure(operation *outcomes.Operation, err error) {
operation.Cause = err
operation.FailureClass = FailureFatal
operation.FailureReason = outcomes.ReasonInternalFailure
var failure *Failure
if errors.As(err, &failure) {
operation.FailureClass = failure.Class
if failure.Reason != "" {
operation.FailureReason = failure.Reason
}
}
}
func IsMalformedError(err error) bool {
var failure *Failure
return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput
return errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted
}

View File

@@ -9,11 +9,11 @@ import (
func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) {
cause := errors.New("provider token secret-value")
malformed := &Failure{Kind: FailureMalformedOutput, Message: "malformed structured output", Cause: cause}
malformed := &Failure{Class: FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "malformed structured output", Cause: cause}
if !IsMalformedError(malformed) || !errors.Is(malformed, cause) || malformed.Error() != "malformed structured output" {
t.Fatalf("malformed classification: %v", malformed)
}
runtime := &Failure{Kind: FailureRuntime, Message: "structured model call failed", Cause: cause}
runtime := &Failure{Class: FailureFatal, Message: "structured model call failed", Cause: cause}
if IsMalformedError(runtime) || IsMalformedError(cause) || IsMalformedError(nil) || !errors.Is(runtime, cause) || runtime.Error() != "structured model call failed" {
t.Fatalf("runtime classification: %v", runtime)
}

View File

@@ -46,7 +46,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{
mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"),
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
cause := errors.New("underlying transport error")
failed := &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed", Cause: cause}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
@@ -67,6 +68,10 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 {
t.Fatalf("batch operations: %+v", result.Operations)
}
operation := result.Operations[1]
if operation.FailureClass != outcomes.FailureRecoverable || !errors.Is(operation.Cause, cause) {
t.Fatalf("validation lost typed failure facts: %+v", operation)
}
if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil {
t.Fatal(err)
}

View File

@@ -3,7 +3,6 @@ package validators
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
@@ -103,6 +102,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
operations = append(operations, outcomes.Operation{
Reference: operationRef(outcomes.OperationValidationInput, nil),
FailureClass: outcomes.FailureRecoverable,
FailureReason: outcomes.ReasonValidationInputTooLarge,
ProposalIndices: indices,
})
@@ -173,17 +173,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
}
if err != nil {
artifacts := writeInteraction(nil, err)
var failure *structuredoutput.Failure
if errors.As(err, &failure) {
operation.FailureReason = failure.Reason
}
if operation.FailureReason == "" {
operation.FailureReason = outcomes.ReasonInternalFailure
}
structuredoutput.RecordFailure(&operation, err)
operation.ProposalIndices = batchProposalIndices(batch.Items)
operations = append(operations, operation)
if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" {
decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable")
if operation.FailureClass == outcomes.FailureRecoverable {
decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable")
for i := range decisions {
decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath
}
@@ -196,6 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
reconciled := reconcileLLMResponse(batch.Items, response)
artifacts := writeInteraction(&reconciled, nil)
if len(reconciled.UnavailableIndices) > 0 {
operation.FailureClass = outcomes.FailureRecoverable
operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid
operation.ProposalIndices = reconciled.UnavailableIndices
} else {

View File

@@ -191,7 +191,7 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
}
func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client