diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 36dd701..e222313 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -1206,6 +1206,30 @@ func TestSubprocessInfeasibleSectionTargetIsAnInputFailure(t *testing.T) { } } +func TestSubprocessMissingCredentialIdentifiesRequiredEnvironmentVariable(t *testing.T) { + f := newProcessFixture(t, "success", "[grammar]") + f.provider.secret = "" + resources := filepath.Join(f.dir, "prompts.yml") + body, err := os.ReadFile(resources) + if err != nil { + t.Fatal(err) + } + body = bytes.Replace(body, []byte(" - id: proposal\n"), []byte(" - id: proposal\n api_key_required: true\n api_key_env: AUDITA_TEST_PROVIDER_KEY\n"), 1) + if err := os.WriteFile(resources, body, 0600); err != nil { + t.Fatal(err) + } + code, stdout, stderr := f.run(t, true) + if code != 1 || stdout != "" || f.provider.callCount() != 0 || + !strings.Contains(stderr, "AUDITA_TEST_PROVIDER_KEY") || !strings.Contains(stderr, `proposal profile "proposal"`) { + t.Fatalf("missing credential: status=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount()) + } + document := f.errors(t) + if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || + document.Errors[0].Reason != "setup_failure" || strings.Contains(document.Errors[0].Message, "AUDITA_TEST_PROVIDER_KEY") { + t.Fatalf("expected compact setup failure in ordinary report: %+v", document) + } +} + func TestSubprocessUnknownProfileFailsBeforeModelDispatchAndOutput(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, stdout, stderr := f.run(t, true, "--proposal-profile", "unknown") diff --git a/docs/internal/llm-runtime.md b/docs/internal/llm-runtime.md index 0b33dc0..01e123a 100644 --- a/docs/internal/llm-runtime.md +++ b/docs/internal/llm-runtime.md @@ -12,10 +12,14 @@ profiles and effective budgets. Clients share engine-local backend admission. The adapter uses ordinary `Run` with `OperationTimeout` and `OnPrepared`, passes Audita's per-call schema and repair budget, and decodes only validated results. The operation budget includes preparation, admission, retries, and structural -repair. `ErrOperationTimeout` remains recoverable even when wrapped by a -preparation error. Parent cancellation remains distinct. The prepared notification +repair. An expired parent context stops processing and retains any native cause. +With an active parent, PromptKit's operation deadline remains recoverable even +when a preparation error wraps it. A native transport timeout affects the HTTP +attempt and is recorded as a transport failure. The prepared notification provides the actual request snapshot; failures before notification have no snapshot. -PromptKit owns backend and credential selection. See the [integration contract](../integrations/openai-compatible-llm.md) +PromptKit owns backend and credential selection. When a required credential is absent, +the CLI identifies the role, profile, and environment variable using the native +diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md) for failure behavior and the [configuration reference](../config.md) for sources. When correction repair is enabled, the proposal call captures a native immutable @@ -26,7 +30,10 @@ for assessment. Follow-ups select the repair prompt, profile, and output schema directly; Audita does not rerender the original proposal or assemble replay messages. An opaque exchange in the structured-output contract keeps provider state outside modules and validators. The adapter shares request construction, notifications, -observers, and result/failure finalization between ordinary and coordinated calls. +observers, and once-only completion between ordinary and coordinated calls. +Completion attaches terminal counts when available and records one logical finish. +Prepared requests and terminal output travel in the returned response for an +optional interaction artifact. Logical call notifications carry metadata. Audita owns actionable rejection, revision decoding, withdrawal, current previews, and full revalidation. Accepted siblings remain independent. Requesting more work @@ -44,7 +51,11 @@ selected runs in a native `ExecutionGroup`. Proposal, repair, and nested validat clients explicitly share the invocation observer; history reuse adds no extra run. Native summaries deduplicate terminal totals by run ID and retain terminal coverage separately from usage presence and completeness. Logical calls and run membership -can differ, especially on failures before execution. Ordinary -reports use compact safe outcomes; request and response evidence is optional -debug material. The runner preserves ordered pipeline traversal and deterministic -section collection while canceling and joining peers on fatal errors. +can differ, especially on failures before execution. + +Processing operations retain the failure class, reason, and underlying error. +When a client omits the reason, the operation uses `internal_failure` and keeps +the original class and error chain. Ordinary reports use compact outcomes. +Request and response evidence is optional debug material. The runner preserves +ordered pipeline traversal and deterministic section collection while canceling +and joining peers on fatal errors. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5d35db4..6d2cb3b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -447,6 +447,14 @@ controlled budgets. Check setup class/cause and useful native information, plus ordinary/coordinator terminal and undelivered-evidence paths. Update [LLM runtime internals](../internal/llm-runtime.md). +**Complete.** Ordinary and coordinated calls share once-only completion, +including prepared and undelivered evidence and native terminal counts. +Proposal, validation, and repair retain typed failure facts. Independent +parent, operation, and attempt deadlines produced the expected classifications. +The CLI identifies a missing required credential by role, profile, and +environment variable while the ordinary report retains the setup reason. +The CLI build, full offline Go suite, and targeted race checks passed. + ## Stage 13 — Use one proposal handoff Audit #46. diff --git a/internal/cli/process_pipeline.go b/internal/cli/process_pipeline.go index 168394d..d2e0a43 100644 --- a/internal/cli/process_pipeline.go +++ b/internal/cli/process_pipeline.go @@ -68,7 +68,9 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure} var cause *structuredoutput.Failure if errors.As(runErr, &cause) { - failure.Reason = cause.Reason + if cause.Reason != "" { + failure.Reason = cause.Reason + } } else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) { failure.Reason = outcomes.ReasonCanceled } diff --git a/internal/framework/llm/adapter.go b/internal/framework/llm/adapter.go index bde12b2..b78b15b 100644 --- a/internal/framework/llm/adapter.go +++ b/internal/framework/llm/adapter.go @@ -17,7 +17,7 @@ import ( ) // CallHooks receive one logical start and finish record and the execution -// events for this call. Call records can contain transcript content. +// events associated with the call. type CallHooks struct { OnCall func(structuredoutput.CallRecord) OnEvent pk.ExecutionObserver @@ -45,35 +45,25 @@ func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hoo } type pendingCall struct { - request pk.RunRequest - finish func(*pk.RunResult, error) (structuredoutput.Response, error) + request pk.RunRequest + once sync.Once + complete func(*pk.RunResult, error) (structuredoutput.Response, error) + response structuredoutput.Response + err error +} + +func (p *pendingCall) finish(result *pk.RunResult, err error) (structuredoutput.Response, error) { + p.once.Do(func() { + p.response, p.err = p.complete(result, err) + }) + return p.response, p.err } func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) { - var response structuredoutput.Response - var err error - var finishHooks []func() - ctx := parent - var terminalMu sync.Mutex + var mu sync.Mutex var terminal pk.ExecutionEvent - terminalAvailable := false - finishHooks = append(finishHooks, func() { - terminalMu.Lock() - if terminalAvailable { - response.Execution = outcomes.ExecutionCounts{ - Available: true, - ProviderAttempts: int(terminal.Attempts), - TransientRetries: int(terminal.TransientRetries), - StructuralRepairs: int(terminal.RepairCalls), - RetriesExhausted: terminal.RetriesExhausted, - } - if terminal.StatusCode != 0 { - status := terminal.StatusCode - response.Execution.HTTPStatus = &status - } - } - terminalMu.Unlock() - }) + var terminalAvailable bool + var prepared *structuredoutput.PreparedSnapshot var record structuredoutput.CallRecord if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil { record.CallID = c.runtime.nextCallID.Add(1) @@ -85,40 +75,61 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re record.Started = time.Now().UTC() record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID} onCall(record) - finishHooks = append(finishHooks, func() { + } + call := &pendingCall{complete: func(result *pk.RunResult, runErr error) (structuredoutput.Response, error) { + var response structuredoutput.Response + mu.Lock() + if terminalAvailable { + response.Execution = outcomes.ExecutionCounts{ + Available: true, ProviderAttempts: int(terminal.Attempts), + TransientRetries: int(terminal.TransientRetries), StructuralRepairs: int(terminal.RepairCalls), + RetriesExhausted: terminal.RetriesExhausted, + } + if terminal.StatusCode != 0 { + status := terminal.StatusCode + response.Execution.HTTPStatus = &status + } + } + if prepared != nil { + response.Prepared = prepared + response.Prompt = prepared.Prompt + response.Model = prepared.Prompt.Model + } + mu.Unlock() + if result != nil { + response.FinalContent = result.RawOutput + response.Model = result.ModelName + } + err := c.finishResponse(parent, &response, result, runErr, out) + if onCall := c.hooks.OnCall; onCall != nil { + if response.Prepared != nil { + record.Prompt = response.Prompt + } record.Finished = time.Now().UTC() if err != nil { - var failure *structuredoutput.Failure - if errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted { + record.Failure = structuredoutput.FailureRuntime + if structuredoutput.IsMalformedError(err) { record.Failure = structuredoutput.FailureMalformedOutput - } else { - record.Failure = structuredoutput.FailureRuntime } } onCall(record) - }) - } - finish := func() { - for i := len(finishHooks) - 1; i >= 0; i-- { - finishHooks[i]() } + return response, err + }} + fail := func(err error) (*pendingCall, error) { + _, err = call.finish(nil, err) + return call, err } - fail := func(failure error) (*pendingCall, error) { - err = failure - finish() - return nil, failure + if err := parent.Err(); err != nil { + return fail(err) } - if parent.Err() != nil { - return fail(classifiedFailure(parent, ctx, parent.Err(), nil)) - } - if err = validateOutputTarget(out); err != nil { + if err := validateOutputTarget(out); err != nil { return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}) } - contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts) - if contractErr != nil { - return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}) + contract, err := OutputContract(req.Schema, c.settings.RepairAttempts) + if err != nil { + return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: err}) } - vars := prompts.ProcessingVariables(req.Variables) var exchange *pk.Exchange if req.ContinueFrom != nil { var ok bool @@ -127,75 +138,68 @@ func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Re return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"}) } } - runRequest := pk.RunRequest{ - PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars, + call.request = pk.RunRequest{ + PromptID: req.PromptID, ProfileID: c.profileID, Vars: prompts.ProcessingVariables(req.Variables), CaptureExchange: req.CaptureExchange, ContinueFrom: exchange, - TransientRetries: &c.settings.TransientRetries, - Validation: contract, + TransientRetries: &c.settings.TransientRetries, OperationTimeout: c.settings.OperationTimeout, + Validation: contract, Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) { if event.Kind == pk.ExecutionFinished { - terminalMu.Lock() + mu.Lock() terminal = event terminalAvailable = true - terminalMu.Unlock() + mu.Unlock() } if c.hooks.OnEventWithCall != nil { c.hooks.OnEventWithCall(record.CallID, record.Context, event) } }), + OnPrepared: func(_ context.Context, details pk.PreparedRun) { + snapshot := preparedSnapshot(details) + mu.Lock() + prepared = snapshot + mu.Unlock() + }, } - var details pk.PreparedRun - capturePrepared := func(_ context.Context, prepared pk.PreparedRun) { - details = prepared - response.Prompt = promptMetadata(details) - record.Prompt = response.Prompt - response.Model = details.EffectiveModelParams.Model - response.Prepared = preparedSnapshot(details) - record.Prepared = response.Prepared + return call, nil +} + +func (c *RoleClient) finishResponse(parent context.Context, response *structuredoutput.Response, result *pk.RunResult, runErr error, out any) error { + if runErr != nil || parent.Err() != nil { + var typed *structuredoutput.Failure + if parent.Err() == nil && errors.As(runErr, &typed) { + return runErr + } + failure := classifiedFailure(parent, runErr, result) + if failure.Reason == outcomes.ReasonSetupFailure && errors.Is(failure, pk.ErrAPIKeyEnvMissing) { + failure.Message = fmt.Sprintf("structured model setup failed for %s profile %q: %v", c.role, c.profileID, failure.Cause) + } + return failure } - runRequest.OperationTimeout = c.settings.OperationTimeout - runRequest.OnPrepared = capturePrepared - return &pendingCall{request: runRequest, finish: func(result *pk.RunResult, runErr error) (completed structuredoutput.Response, finalErr error) { - defer func() { err = finalErr; finish(); completed = response }() - - if result != nil { - response.FinalContent = result.RawOutput - record.FinalContent = result.RawOutput - response.Model = result.ModelName - response.PromptTokens = result.Usage.PromptTokens - response.CompletionTokens = result.Usage.CompletionTokens - response.TotalTokens = result.Usage.TotalTokens + if result == nil { + return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"} + } + if !result.Validation.IsValid { + return classifiedFailure(parent, nil, result) + } + if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil { + var typeError *json.UnmarshalTypeError + if errors.As(decodeErr, &typeError) { + return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr} } - if runErr != nil { - return response, classifiedFailure(parent, ctx, runErr, result) - } - if result == nil { - return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"} - } - if !result.Validation.IsValid { - return response, classifiedFailure(parent, ctx, nil, result) - } - if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil { - var typeError *json.UnmarshalTypeError - if errors.As(decodeErr, &typeError) { - return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr} - } - return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr} - } - if result.Exchange != nil { - response.Exchange = &structuredoutput.Exchange{State: result.Exchange} - } - - response.Content = json.RawMessage(result.RawOutput) - response.Provider = "openai-compatible" - return response, nil - }}, nil + return &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr} + } + if result.Exchange != nil { + response.Exchange = &structuredoutput.Exchange{State: result.Exchange} + } + response.Content = json.RawMessage(result.RawOutput) + return nil } func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) { call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent) if err != nil { - return structuredoutput.Response{}, err + return call.response, err } result, err := c.runtime.Engine.Run(ctx, call.request) return call.finish(result, err) @@ -212,13 +216,15 @@ func OutputContract(schema responseschema.Key, repairs int) (*pk.OutputContract, // classifiedFailure deliberately checks setup defects before broad PromptKit // validation/generation wrappers. Provider text is kept only in the typed cause. -func classifiedFailure(parent, local context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure { +func classifiedFailure(parent context.Context, cause error, result *pk.RunResult) *structuredoutput.Failure { failure := &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call failed", Cause: cause} if parent.Err() != nil { failure.Class = structuredoutput.FailureCanceled failure.Reason = outcomes.ReasonCanceled failure.Message = "processing canceled" - failure.Cause = parent.Err() + if !errors.Is(cause, parent.Err()) { + failure.Cause = errors.Join(parent.Err(), cause) + } return failure } if errors.Is(cause, pk.ErrOperationTimeout) { @@ -232,7 +238,13 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru failure.Message = "structured model setup failed" return failure } - if errors.Is(local.Err(), context.DeadlineExceeded) || errors.Is(cause, context.DeadlineExceeded) { + if errors.Is(cause, pk.ErrTransport) { + failure.Class = structuredoutput.FailureRecoverable + failure.Reason = outcomes.ReasonTransportFailed + failure.Message = "model transport failed" + return failure + } + if errors.Is(cause, context.DeadlineExceeded) { failure.Class = structuredoutput.FailureRecoverable failure.Reason = outcomes.ReasonOperationTimeout failure.Message = "model operation timed out" @@ -244,12 +256,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru failure.Message = "model capacity was unavailable" return failure } - if errors.Is(cause, pk.ErrTransport) { - failure.Class = structuredoutput.FailureRecoverable - failure.Reason = outcomes.ReasonTransportFailed - failure.Message = "model transport failed" - return failure - } if errors.Is(cause, pk.ErrMalformedResponse) { failure.Class = structuredoutput.FailureRecoverable failure.Reason = outcomes.ReasonProviderResponseInvalid @@ -265,7 +271,6 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru } if result != nil && !result.Validation.IsValid { failure.Class = structuredoutput.FailureRecoverable - failure.Kind = structuredoutput.FailureMalformedOutput failure.Reason = outcomes.ReasonInvalidOutputExhausted failure.Message = "structured output remained invalid" return failure diff --git a/internal/framework/llm/adapter_test.go b/internal/framework/llm/adapter_test.go index 0d6e8f8..81f39c7 100644 --- a/internal/framework/llm/adapter_test.go +++ b/internal/framework/llm/adapter_test.go @@ -38,7 +38,7 @@ func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs in var count atomic.Int32 t.Setenv("ADAPTER_KEY", "adapter-secret") dir := t.TempDir() - resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n" + resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n api_key_required: true\n api_key_env: ADAPTER_KEY\n model: full-model\n - id: short-attempt\n backend: offline\n model: full-model\n timeout_seconds: 1\n" for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} { if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil { t.Fatal(err) @@ -170,13 +170,13 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) { if len(records) != 2 || records[0].Finished.IsZero() == false || records[1].Finished.IsZero() || records[1].Context.Role != role { t.Fatalf("logical records: %+v", records) } - if tc.malformed && records[1].FinalContent == "" { + if tc.malformed && (response.FinalContent == "" || len(response.Content) != 0) { t.Fatal("invalid final content lost") } if len(events) == 0 || events[len(events)-1].Kind != pk.ExecutionFinished || int(events[len(events)-1].RepairCalls) != tc.repairCalls || int(events[len(events)-1].TransientRetries) != tc.transientRetries { t.Fatalf("execution counts: %+v", events) } - if err == nil && (out == nil || string(response.Content) != valid || response.TotalTokens != 5*(tc.repairCalls+1) || response.Model != "full-model") { + if err == nil && (out == nil || string(response.Content) != valid || response.Model != "full-model") { t.Fatalf("decoded response: %+v, %v", response, out) } }) @@ -404,7 +404,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) { } func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) { - blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) } + blocking := func(r *http.Request) { <-r.Context().Done() } runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking) runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) @@ -414,7 +414,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t var output map[string]any response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) var failure *structuredoutput.Failure - if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !response.Execution.Available { + if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(err, pk.ErrOperationTimeout) || !response.Execution.Available { t.Fatalf("local timeout: failure=%#v counts=%+v", failure, response.Execution) } @@ -426,6 +426,58 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t } } +func TestPreparedAdapterDistinguishesAttemptTimeoutFromParentDeadline(t *testing.T) { + blocking := func(r *http.Request) { <-r.Context().Done() } + runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking) + client, err := runtime.NewRoleClient("proposal", "short-attempt", runtime.Config.LLM, CallHooks{}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + parentBudget time.Duration + class outcomes.FailureClass + reason outcomes.ReasonCode + }{ + {"attempt", 5 * time.Second, outcomes.FailureRecoverable, outcomes.ReasonTransportFailed}, + {"parent", 300 * time.Millisecond, outcomes.FailureCanceled, outcomes.ReasonCanceled}, + } { + t.Run(tc.name, func(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), tc.parentBudget) + defer cancel() + before := count.Load() + var output map[string]any + response, err := client.CompleteStructured(parent, adapterRequest(responseschema.CorrectionSetKey), &output) + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Class != tc.class || failure.Reason != tc.reason || + !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, pk.ErrOperationTimeout) || + !response.Execution.Available || count.Load() != before+1 { + t.Fatalf("deadline classification: failure=%#v error=%v counts=%+v requests=%d", failure, err, response.Execution, count.Load()-before) + } + if tc.class == outcomes.FailureRecoverable && (!errors.Is(err, pk.ErrTransport) || parent.Err() != nil) { + t.Fatalf("missing transport cause or parent also expired: error=%v parent=%v", err, parent.Err()) + } + }) + } +} + +func TestPreparedAdapterMissingCredentialRetainsSetupCause(t *testing.T) { + runtime, count, _ := adapterFixture(t, nil, 0, 0, nil) + t.Setenv("ADAPTER_KEY", "") + client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{}) + if err != nil { + t.Fatal(err) + } + var output map[string]any + _, err = client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &output) + var failure *structuredoutput.Failure + if !errors.As(err, &failure) || failure.Class != outcomes.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || + !errors.Is(err, pk.ErrAPIKeyEnvMissing) || !strings.Contains(err.Error(), "ADAPTER_KEY") || + !strings.Contains(err.Error(), `proposal profile "base"`) || count.Load() != 0 { + t.Fatalf("missing credential classification: failure=%#v error=%v requests=%d", failure, err, count.Load()) + } +} + func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) { runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"reason":"ok"}]}`}}, 0, 0, nil) client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{}) @@ -477,7 +529,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) { parent, cancel := context.WithCancel(context.Background()) defer cancel() cause := fmt.Errorf("%w: embedded contract defect", pk.ErrValidation) - failure := classifiedFailure(parent, parent, cause, nil) + failure := classifiedFailure(parent, cause, nil) if failure.Class != structuredoutput.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure || !errors.Is(failure, pk.ErrValidation) { t.Fatalf("unknown validation failure: %#v", failure) } @@ -488,7 +540,7 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) { func TestOperationTimeoutDuringPreparationIsRecoverable(t *testing.T) { ctx := context.Background() cause := errors.Join(pk.ErrPromptLoad, pk.ErrOperationTimeout, context.DeadlineExceeded) - failure := classifiedFailure(ctx, ctx, cause, nil) + failure := classifiedFailure(ctx, cause, nil) if failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(failure, pk.ErrOperationTimeout) { t.Fatalf("failure = %#v", failure) } diff --git a/internal/framework/llm/coordinate.go b/internal/framework/llm/coordinate.go index ade437b..28035a1 100644 --- a/internal/framework/llm/coordinate.go +++ b/internal/framework/llm/coordinate.go @@ -36,14 +36,15 @@ func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exc return &pk.RunRequest{}, nil } var output json.RawMessage - pending, err = c.prepareCall(ctx, *selected, &output, nil) + call, err := c.prepareCall(ctx, *selected, &output, nil) if err != nil { - _, domainErr := next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: err}) + _, domainErr := next(ctx, structuredoutput.Turn{Response: &call.response, Err: err}) if domainErr != nil { return nil, domainErr } return nil, err } + pending = call return &pending.request, nil }, }) diff --git a/internal/framework/outcomes/outcomes.go b/internal/framework/outcomes/outcomes.go index cc50bcf..f824411 100644 --- a/internal/framework/outcomes/outcomes.go +++ b/internal/framework/outcomes/outcomes.go @@ -41,8 +41,16 @@ type ExecutionCounts struct { HTTPStatus *int `json:"http_status,omitempty"` } -// ReasonCode is a fixed, safe explanation category. Raw provider errors and -// prompt content are never stored in an operation outcome. +// FailureClass determines whether processing can skip an operation or must stop. +type FailureClass string + +const ( + FailureRecoverable FailureClass = "recoverable" + FailureFatal FailureClass = "fatal" + FailureCanceled FailureClass = "canceled" +) + +// ReasonCode is a fixed explanation category for the serialized outcome. type ReasonCode string const ( @@ -85,6 +93,8 @@ type Operation struct { Reference Reference `json:"reference"` Counts ExecutionCounts `json:"counts"` Succeeded bool `json:"succeeded"` + FailureClass FailureClass `json:"-"` + Cause error `json:"-"` FailureReason ReasonCode `json:"failure_reason,omitempty"` RecoveryReasons []RecoveryReason `json:"recovery_reasons,omitempty"` ProposalIndices []int `json:"proposal_indices,omitempty"` diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index ecf46b0..434cf75 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -5,7 +5,6 @@ package proposal_generation import ( "context" - "errors" "fmt" "strings" @@ -129,15 +128,9 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } if callErr != nil { - var failure *structuredoutput.Failure - if errors.As(callErr, &failure) { - operation.FailureReason = failure.Reason - } - if operation.FailureReason == "" { - operation.FailureReason = outcomes.ReasonInternalFailure - } + structuredoutput.RecordFailure(&operation, callErr) result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts} - if structuredoutput.IsRecoverable(callErr) && failure != nil && failure.Reason != "" { + if operation.FailureClass == outcomes.FailureRecoverable { return result, nil } return result, fmt.Errorf("proposal generation completion failed: %w", callErr) diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index e244f52..06afa42 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -226,6 +226,20 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te } } +func TestGenerateCandidatesUsesInternalReasonForUnspecifiedRecoverableFailure(t *testing.T) { + cause := errors.New("underlying call failure") + req := defaultRequest(t) + req.LLMClient = &fakeStructuredClient{err: &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Cause: cause}} + result, err := GenerateCandidates(context.Background(), req) + if err != nil || len(result.Operations) != 1 { + t.Fatalf("recoverable proposal returned operations=%+v error=%v", result.Operations, err) + } + operation := result.Operations[0] + if operation.FailureClass != outcomes.FailureRecoverable || operation.FailureReason != outcomes.ReasonInternalFailure || !errors.Is(operation.Cause, cause) { + t.Fatalf("operation lost typed failure facts: %+v", operation) + } +} + func TestGenerateCandidatesFatalSetupPreservesCallFacts(t *testing.T) { client := &fakeStructuredClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "structured model setup failed", Cause: errors.New("broken prompt")}, completion: structuredoutput.Response{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 0}}} req := defaultRequest(t) diff --git a/internal/framework/runner/repair.go b/internal/framework/runner/repair.go index 8996394..f773fdb 100644 --- a/internal/framework/runner/repair.go +++ b/internal/framework/runner/repair.go @@ -2,7 +2,6 @@ package runner import ( "context" - "errors" "fmt" "slices" @@ -174,14 +173,9 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, branch.Attempts = append(branch.Attempts, record) } if callErr != nil { - var failure *structuredoutput.Failure - if errors.As(callErr, &failure) { - operation.FailureReason = failure.Reason - } else { - operation.FailureReason = outcomes.ReasonInternalFailure - } + structuredoutput.RecordFailure(&operation, callErr) finishAttempt(callErr, nil, validateSectionCandidatesResult{}) - if structuredoutput.IsRecoverable(callErr) { + if operation.FailureClass == outcomes.FailureRecoverable { branch.Outcome = "unavailable" return nil, nil } @@ -190,7 +184,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput, var err error response, err = proposal_generation.DecodeRepairResponse(completion.Content, position+1) if err != nil { - operation.FailureReason = outcomes.ReasonInvalidOutput + structuredoutput.RecordFailure(&operation, err) finishAttempt(err, nil, validateSectionCandidatesResult{}) branch.Outcome = "unavailable" return nil, nil diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index ec15e2b..9392f0d 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -189,7 +189,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) { {"exhausted", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`}, nil, 2, "exhausted", 2}, {"no-effect revision", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}}`}, nil, 2, "rejected", 1}, {"wrong reference", []string{`{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "unavailable", 1}, - {"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1}, + {"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed", Cause: errors.New("underlying provider error")}, 1, "unavailable", 1}, } { t.Run(tc.name, func(t *testing.T) { client := &repairClient{t: t, outputs: tc.outputs, err: tc.err} @@ -199,13 +199,20 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) { if err != nil { t.Fatal(err) } - _, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") + result, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair") if err != nil { t.Fatal(err) } if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 { t.Fatalf("branches=%+v calls=%d", branches, len(client.calls)) } + if tc.err != nil { + operation := result.operations[len(result.operations)-1] + if operation.FailureClass != outcomes.FailureRecoverable || + operation.FailureReason != outcomes.ReasonProviderFailed || !errors.Is(operation.Cause, tc.err) { + t.Fatalf("repair lost typed failure facts: %+v", operation) + } + } if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil { t.Fatalf("history: %+v", client.calls[1].ContinueFrom) } diff --git a/internal/framework/structuredoutput/call.go b/internal/framework/structuredoutput/call.go index 2a2786e..4ac18e2 100644 --- a/internal/framework/structuredoutput/call.go +++ b/internal/framework/structuredoutput/call.go @@ -50,17 +50,13 @@ type Exchange struct { } type Response struct { - Exchange *Exchange `json:"-"` - Execution outcomes.ExecutionCounts `json:"execution"` - Content json.RawMessage `json:"content,omitempty"` - Provider string `json:"provider,omitempty"` - Model string `json:"model,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - Prompt PromptMetadata `json:"prompt"` - Prepared *PreparedSnapshot `json:"prepared,omitempty"` - FinalContent string `json:"final_content,omitempty"` + Exchange *Exchange `json:"-"` + Execution outcomes.ExecutionCounts `json:"execution"` + Content json.RawMessage `json:"content,omitempty"` + Model string `json:"model,omitempty"` + Prompt PromptMetadata `json:"prompt"` + Prepared *PreparedSnapshot `json:"prepared,omitempty"` + FinalContent string `json:"final_content,omitempty"` } // PreparedSnapshot is the exact rendered request snapshot. Its messages may @@ -81,8 +77,7 @@ type CacheControl struct { TTL string `json:"ttl,omitempty"` } -// CallRecord brackets one logical operation, including preparation. Content -// is present only when a model returned it; consumers must protect it. +// CallRecord brackets one logical operation, including preparation. type CallRecord struct { RequestedPromptID string `json:"requested_prompt_id,omitempty"` CallID uint64 `json:"call_id"` @@ -90,8 +85,6 @@ type CallRecord struct { Prompt PromptMetadata `json:"prompt"` Started time.Time `json:"started"` Finished time.Time `json:"finished,omitempty"` - Prepared *PreparedSnapshot `json:"prepared,omitempty"` - FinalContent string `json:"final_content,omitempty"` Failure FailureKind `json:"failure,omitempty"` } diff --git a/internal/framework/structuredoutput/failure.go b/internal/framework/structuredoutput/failure.go index 73e9e75..edd0219 100644 --- a/internal/framework/structuredoutput/failure.go +++ b/internal/framework/structuredoutput/failure.go @@ -14,10 +14,8 @@ const ( FailureRuntime FailureKind = "runtime" ) -// Failure exposes a safe message while retaining the typed cause for callers. -// The cause must not be serialized or formatted directly in user artifacts. +// Failure exposes an operator-facing message and retains the typed cause. type Failure struct { - Kind FailureKind // Retained until the adapter switches to reason-based classification. Class FailureClass Reason outcomes.ReasonCode Message string @@ -26,12 +24,12 @@ type Failure struct { // FailureClass determines whether domain processing may safely skip a model // operation or must stop the run. Parent cancellation always stops the run. -type FailureClass string +type FailureClass = outcomes.FailureClass const ( - FailureRecoverable FailureClass = "recoverable" - FailureFatal FailureClass = "fatal" - FailureCanceled FailureClass = "canceled" + FailureRecoverable = outcomes.FailureRecoverable + FailureFatal = outcomes.FailureFatal + FailureCanceled = outcomes.FailureCanceled ) func (f *Failure) Error() string { return f.Message } @@ -42,7 +40,22 @@ func IsRecoverable(err error) bool { return errors.As(err, &failure) && failure.Class == FailureRecoverable } +// RecordFailure retains the error chain for processing and sets the compact +// reason used by reports. An absent reason does not change recoverability. +func RecordFailure(operation *outcomes.Operation, err error) { + operation.Cause = err + operation.FailureClass = FailureFatal + operation.FailureReason = outcomes.ReasonInternalFailure + var failure *Failure + if errors.As(err, &failure) { + operation.FailureClass = failure.Class + if failure.Reason != "" { + operation.FailureReason = failure.Reason + } + } +} + func IsMalformedError(err error) bool { var failure *Failure - return errors.As(err, &failure) && failure.Kind == FailureMalformedOutput + return errors.As(err, &failure) && failure.Reason == outcomes.ReasonInvalidOutputExhausted } diff --git a/internal/framework/structuredoutput/failure_test.go b/internal/framework/structuredoutput/failure_test.go index e6b1b13..5ba0f71 100644 --- a/internal/framework/structuredoutput/failure_test.go +++ b/internal/framework/structuredoutput/failure_test.go @@ -9,11 +9,11 @@ import ( func TestFailureClassificationKeepsCauseAndSafeMessage(t *testing.T) { cause := errors.New("provider token secret-value") - malformed := &Failure{Kind: FailureMalformedOutput, Message: "malformed structured output", Cause: cause} + malformed := &Failure{Class: FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "malformed structured output", Cause: cause} if !IsMalformedError(malformed) || !errors.Is(malformed, cause) || malformed.Error() != "malformed structured output" { t.Fatalf("malformed classification: %v", malformed) } - runtime := &Failure{Kind: FailureRuntime, Message: "structured model call failed", Cause: cause} + runtime := &Failure{Class: FailureFatal, Message: "structured model call failed", Cause: cause} if IsMalformedError(runtime) || IsMalformedError(cause) || IsMalformedError(nil) || !errors.Is(runtime, cause) || runtime.Error() != "structured model call failed" { t.Fatalf("runtime classification: %v", runtime) } diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go index a73b104..3f1201e 100644 --- a/internal/framework/validators/llm_partial_test.go +++ b/internal/framework/validators/llm_partial_test.go @@ -46,7 +46,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { candidates := []proposals.EnrichedCorrectionProposal{ mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures"), } - failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} + cause := errors.New("underlying transport error") + failed := &structuredoutput.Failure{Class: outcomes.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed", Cause: cause} client := &scriptedValidationClient{replies: []validationReply{ {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, @@ -67,6 +68,10 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { if len(result.Operations) != 3 || result.Operations[1].FailureReason != outcomes.ReasonTransportFailed || result.Operations[1].Reference.BatchIndex == nil || *result.Operations[1].Reference.BatchIndex != 1 || result.Operations[1].Counts.ProviderAttempts != 1 { t.Fatalf("batch operations: %+v", result.Operations) } + operation := result.Operations[1] + if operation.FailureClass != outcomes.FailureRecoverable || !errors.Is(operation.Cause, cause) { + t.Fatalf("validation lost typed failure facts: %+v", operation) + } if err := EnforceDecisionCardinality(candidates, result.Decisions); err != nil { t.Fatal(err) } diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index e857f67..618a11d 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -3,7 +3,6 @@ package validators import ( "context" "encoding/json" - "errors" "fmt" "sort" "strings" @@ -103,6 +102,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } operations = append(operations, outcomes.Operation{ Reference: operationRef(outcomes.OperationValidationInput, nil), + FailureClass: outcomes.FailureRecoverable, FailureReason: outcomes.ReasonValidationInputTooLarge, ProposalIndices: indices, }) @@ -173,17 +173,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, } if err != nil { artifacts := writeInteraction(nil, err) - var failure *structuredoutput.Failure - if errors.As(err, &failure) { - operation.FailureReason = failure.Reason - } - if operation.FailureReason == "" { - operation.FailureReason = outcomes.ReasonInternalFailure - } + structuredoutput.RecordFailure(&operation, err) operation.ProposalIndices = batchProposalIndices(batch.Items) operations = append(operations, operation) - if structuredoutput.IsRecoverable(err) && failure != nil && failure.Reason != "" { - decisions := rejectBatch(batch.Items, string(failure.Reason), "model validation was unavailable") + if operation.FailureClass == outcomes.FailureRecoverable { + decisions := rejectBatch(batch.Items, string(operation.FailureReason), "model validation was unavailable") for i := range decisions { decisions[i].DiagnosticArtifactPath = artifacts.ResponsePayloadPath } @@ -196,6 +190,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, reconciled := reconcileLLMResponse(batch.Items, response) artifacts := writeInteraction(&reconciled, nil) if len(reconciled.UnavailableIndices) > 0 { + operation.FailureClass = outcomes.FailureRecoverable operation.FailureReason = outcomes.ReasonValidationDecisionsInvalid operation.ProposalIndices = reconciled.UnavailableIndices } else { diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index 6f2a941..5cfeddb 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -191,7 +191,7 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { } func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) { - client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}} + client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client