Identify built-in generation preparation failures
This commit is contained in:
@@ -198,6 +198,11 @@ fails with the public invalid-request category and zero dispatch. Existing
|
|||||||
provider rejection, missing-key, capacity, injected, and cancellation tests
|
provider rejection, missing-key, capacity, injected, and cancellation tests
|
||||||
retain their respective outcomes.
|
retain their respective outcomes.
|
||||||
|
|
||||||
|
Completion: built-in preparation failures carry direct error and evidence
|
||||||
|
markers; diagnostics return invalid-request errors before dispatch. Generation,
|
||||||
|
use-case, and root tests pass on Go 1.27.3, including the previously deferred
|
||||||
|
injected-client classification cases.
|
||||||
|
|
||||||
## Stage 7: Bound Authentication Recognition And Report Overflow
|
## Stage 7: Bound Authentication Recognition And Report Overflow
|
||||||
|
|
||||||
**Catalog:** 2. **Files:** current authentication probe, public/domain smoke reasons.
|
**Catalog:** 2. **Files:** current authentication probe, public/domain smoke reasons.
|
||||||
|
|||||||
@@ -85,6 +85,12 @@ func (c *Coordinator) GenerateWithEvidence(
|
|||||||
return c.generate(ctx, req, true)
|
return c.generate(ctx, req, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparationError identifies a built-in request preparation failure.
|
||||||
|
type PreparationError struct{ Cause error }
|
||||||
|
|
||||||
|
func (e *PreparationError) Error() string { return e.Cause.Error() }
|
||||||
|
func (e *PreparationError) Unwrap() error { return e.Cause }
|
||||||
|
|
||||||
func (c *Coordinator) generate(
|
func (c *Coordinator) generate(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
@@ -95,7 +101,7 @@ func (c *Coordinator) generate(
|
|||||||
}
|
}
|
||||||
prepared, err := c.builtIn.PrepareGeneration(req)
|
prepared, err := c.builtIn.PrepareGeneration(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err, GenerationEvidence{}
|
return nil, &PreparationError{Cause: err}, GenerationEvidence{PreparationFailed: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
evidence := GenerationEvidence{}
|
evidence := GenerationEvidence{}
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
// GenerationEvidence contains invocation-local request evidence without raw
|
// GenerationEvidence contains invocation-local request evidence without raw
|
||||||
// provider bodies, headers, credentials, or generated content.
|
// provider bodies, headers, credentials, or generated content.
|
||||||
type GenerationEvidence struct {
|
type GenerationEvidence struct {
|
||||||
|
// PreparationFailed identifies a built-in request rejected before attempts.
|
||||||
|
PreparationFailed bool
|
||||||
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
// InitialEndpoint is the frozen built-in request URL, if prepared.
|
||||||
// Injected generation leaves it unknown.
|
// Injected generation leaves it unknown.
|
||||||
InitialEndpoint string
|
InitialEndpoint string
|
||||||
|
|||||||
@@ -498,7 +498,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
repairer: &fakeRepairer{err: repairInvalidRequest},
|
repairer: &fakeRepairer{err: repairInvalidRequest},
|
||||||
wantError: ErrInvalidRequest,
|
wantError: ErrLLMGenerate,
|
||||||
wantSource: repairInvalidRequest,
|
wantSource: repairInvalidRequest,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/capacity"
|
"go.maximumdirect.net/promptkit/internal/capacity"
|
||||||
"go.maximumdirect.net/promptkit/internal/defaults"
|
"go.maximumdirect.net/promptkit/internal/defaults"
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/generation"
|
||||||
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
"go.maximumdirect.net/promptkit/internal/jsonvalue"
|
||||||
"go.maximumdirect.net/promptkit/internal/llm"
|
"go.maximumdirect.net/promptkit/internal/llm"
|
||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
@@ -269,6 +270,9 @@ func (r *Runner) executePreparedRun(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func wrapGenerationError(err error) error {
|
func wrapGenerationError(err error) error {
|
||||||
|
if _, ok := err.(*generation.PreparationError); ok {
|
||||||
|
return &OperationError{Category: ErrInvalidRequest, Cause: err}
|
||||||
|
}
|
||||||
return &OperationError{Category: ErrLLMGenerate, Cause: err}
|
return &OperationError{Category: ErrLLMGenerate, Cause: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2074,7 +2074,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
func TestRunnerRunInjectedInvalidRequestRemainsGenerationFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
@@ -2089,11 +2089,8 @@ func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
|||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
})
|
})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrLLMGenerate) || !errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("expected generation error retaining cause, got %v", err)
|
||||||
}
|
|
||||||
if errors.Is(err, ErrLLMGenerate) {
|
|
||||||
t.Fatalf("did not expect ErrLLMGenerate, got %v", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -210,6 +210,9 @@ func (t *SmokeTester) runInference(
|
|||||||
report.Failure = generation.ClassifyFailure(contextErr)
|
report.Failure = generation.ClassifyFailure(contextErr)
|
||||||
return finish(contextErr)
|
return finish(contextErr)
|
||||||
}
|
}
|
||||||
|
if evidence.PreparationFailed && generationErr != nil {
|
||||||
|
return nil, &OperationError{Category: ErrInvalidRequest, Cause: generationErr}
|
||||||
|
}
|
||||||
if generationErr != nil {
|
if generationErr != nil {
|
||||||
report.Failure = generation.ClassifyFailure(generationErr)
|
report.Failure = generation.ClassifyFailure(generationErr)
|
||||||
report.Completed = true
|
report.Completed = true
|
||||||
|
|||||||
29
preparation_error_contract_test.go
Normal file
29
preparation_error_contract_test.go
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBuiltInPreparationFailureIsInvalidRequestWithoutDispatch(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
cfg := simpleConfig("hello")
|
||||||
|
cfg.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++; return nil, errors.New("unexpected dispatch") })}
|
||||||
|
engine, err := promptkit.NewEngine(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
override := &promptkit.ExecutionTargetOverride{ExtraParams: map[string]any{"model": "reserved"}}
|
||||||
|
_, err = engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt", Execution: override})
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidRequest) || calls != 0 {
|
||||||
|
t.Fatalf("run error = %v, calls = %d", err, calls)
|
||||||
|
}
|
||||||
|
report, err := engine.SmokeTest(context.Background(), promptkit.SmokeTestRequest{Kind: promptkit.SmokeTestTextInference, ProfileID: "profile", Execution: override})
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidRequest) || report != nil || calls != 0 {
|
||||||
|
t.Fatalf("smoke = %+v, %v, calls = %d", report, err, calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user