Refactor the single-run HTTP contract
This commit is contained in:
@@ -145,6 +145,8 @@ scaffolding, or conversation and coordination features.
|
|||||||
|
|
||||||
## Stage 4: Refactor the single-run HTTP API
|
## Stage 4: Refactor the single-run HTTP API
|
||||||
|
|
||||||
|
**Status: Complete.**
|
||||||
|
|
||||||
1. Keep `POST /v1/runs` as the single route: the operation is still a run
|
1. Keep `POST /v1/runs` as the single route: the operation is still a run
|
||||||
and there is no parallel version to distinguish. This is a direct
|
and there is no parallel version to distinguish. This is a direct
|
||||||
contract replacement at that path, not a promise of old-client
|
contract replacement at that path, not a promise of old-client
|
||||||
|
|||||||
@@ -14,5 +14,6 @@
|
|||||||
"vars": {
|
"vars": {
|
||||||
"session_date": "2026-05-04"
|
"session_date": "2026-05-04"
|
||||||
},
|
},
|
||||||
"include_raw_output": false
|
"include_raw_output": false,
|
||||||
|
"include_execution_report": false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,36 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderReturnsDetachedBodies(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("original"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reader, err := NewRestrictedArtifactReader(root, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, ref := range []promptkit.ArtifactRef{promptkit.Inline("original"), promptkit.File("input.txt")} {
|
||||||
|
first, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first.Body[0] = 'X'
|
||||||
|
first.Name = "changed"
|
||||||
|
if string(second.Body) != "original" || second.Name == "changed" {
|
||||||
|
t.Fatalf("read results share mutable state: %+v", second)
|
||||||
|
}
|
||||||
|
second.Body[1] = 'Y'
|
||||||
|
if string(first.Body) != "Xriginal" {
|
||||||
|
t.Fatalf("reader retained a body alias: %q", first.Body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
outside := t.TempDir()
|
outside := t.TempDir()
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
|
|
||||||
type runRequestDTO struct {
|
type runRequestDTO struct {
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
|
||||||
ProfileID string `json:"profile_id,omitempty"`
|
ProfileID string `json:"profile_id,omitempty"`
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
Inputs map[string]inputRefDTO `json:"inputs"`
|
Inputs map[string]inputRefDTO `json:"inputs"`
|
||||||
@@ -59,28 +58,21 @@ type artifactDTO struct {
|
|||||||
type metadataDTO struct {
|
type metadataDTO struct {
|
||||||
RunID string `json:"run_id"`
|
RunID string `json:"run_id"`
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version"`
|
|
||||||
PromptHash string `json:"prompt_hash"`
|
PromptHash string `json:"prompt_hash"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
ModelName string `json:"model_name"`
|
|
||||||
Endpoint string `json:"endpoint"`
|
|
||||||
ModelParams modelParamsDTO `json:"model_params"`
|
ModelParams modelParamsDTO `json:"model_params"`
|
||||||
InputHashes map[string]string `json:"input_hashes"`
|
InputHashes map[string]string `json:"input_hashes"`
|
||||||
Usage tokenUsageDTO `json:"usage"`
|
Usage tokenUsageDTO `json:"usage"`
|
||||||
StartTime time.Time `json:"start_time"`
|
StartTime time.Time `json:"start_time"`
|
||||||
EndTime time.Time `json:"end_time"`
|
EndTime time.Time `json:"end_time"`
|
||||||
DurationMS int64 `json:"duration_ms"`
|
DurationMS int64 `json:"duration_ms"`
|
||||||
ValidationMode string `json:"validation_mode"`
|
|
||||||
ValidationStatus string `json:"validation_status"`
|
|
||||||
RepairAttemptsUsed int `json:"repair_attempts_used"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type modelParamsDTO struct {
|
type modelParamsDTO struct {
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
BackendID string `json:"backend_id,omitempty"`
|
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
|
|||||||
@@ -106,38 +106,36 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||||
}
|
}
|
||||||
|
|
||||||
runContext := r.Context()
|
var operationTimeout time.Duration
|
||||||
if req.RunTimeoutSeconds != nil && *req.RunTimeoutSeconds > 0 {
|
if req.RunTimeoutSeconds != nil && *req.RunTimeoutSeconds > 0 {
|
||||||
var cancel context.CancelFunc
|
operationTimeout = time.Duration(*req.RunTimeoutSeconds) * time.Second
|
||||||
runContext, cancel = context.WithTimeout(runContext, time.Duration(*req.RunTimeoutSeconds)*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
}
|
}
|
||||||
mappedRequest := promptkit.RunRequest{
|
mappedRequest := promptkit.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
SessionID: req.SessionID,
|
SessionID: req.SessionID,
|
||||||
Inputs: mappedInputs,
|
Inputs: mappedInputs,
|
||||||
Vars: req.Vars,
|
Vars: req.Vars,
|
||||||
Execution: model,
|
Execution: model,
|
||||||
TransientRetries: req.TransientRetries,
|
TransientRetries: req.TransientRetries,
|
||||||
|
OperationTimeout: operationTimeout,
|
||||||
}
|
}
|
||||||
var collector *executionreport.Collector
|
var collector *executionreport.Collector
|
||||||
if req.IncludeExecutionReport {
|
if req.IncludeExecutionReport {
|
||||||
collector = executionreport.NewCollector()
|
collector = executionreport.NewCollector()
|
||||||
mappedRequest.Observer = collector.Observe
|
mappedRequest.Observer = collector.Observe
|
||||||
}
|
}
|
||||||
res, err := h.runner.Run(runContext, mappedRequest)
|
res, err := h.runner.Run(r.Context(), mappedRequest)
|
||||||
var execution *executionreport.Execution
|
var execution *executionreport.Execution
|
||||||
if collector != nil {
|
if collector != nil {
|
||||||
execution = collector.Snapshot()
|
execution = collector.Snapshot()
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
switch runContext.Err() {
|
switch {
|
||||||
case context.DeadlineExceeded:
|
case errors.Is(err, promptkit.ErrOperationTimeout), errors.Is(err, context.DeadlineExceeded) && errors.Is(r.Context().Err(), context.DeadlineExceeded):
|
||||||
writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes)
|
writeRunFailure(w, http.StatusGatewayTimeout, "deadline_exceeded", "run execution deadline exceeded", execution, h.options.MaxResponseBytes)
|
||||||
return
|
return
|
||||||
case context.Canceled:
|
case errors.Is(err, context.Canceled) && errors.Is(r.Context().Err(), context.Canceled):
|
||||||
writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes)
|
writeRunFailure(w, http.StatusRequestTimeout, "request_canceled", "run request canceled", execution, h.options.MaxResponseBytes)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -159,14 +157,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
Metadata: metadataDTO{
|
Metadata: metadataDTO{
|
||||||
RunID: res.RunID,
|
RunID: res.RunID,
|
||||||
PromptID: res.PromptID,
|
PromptID: res.PromptID,
|
||||||
PromptVersion: res.PromptVersion,
|
|
||||||
PromptHash: res.PromptHash,
|
PromptHash: res.PromptHash,
|
||||||
RenderedPromptHash: res.RenderedPromptHash,
|
RenderedPromptHash: res.RenderedPromptHash,
|
||||||
SelectedProfileID: res.SelectedProfileID,
|
SelectedProfileID: res.SelectedProfileID,
|
||||||
SelectedBackendID: res.SelectedBackendID,
|
SelectedBackendID: res.SelectedBackendID,
|
||||||
SessionID: res.SessionID,
|
SessionID: res.SessionID,
|
||||||
ModelName: res.ModelName,
|
|
||||||
Endpoint: res.Endpoint,
|
|
||||||
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
ModelParams: modelParamsDTOFromExecutionTarget(res.EffectiveModelParams),
|
||||||
InputHashes: res.InputHashes,
|
InputHashes: res.InputHashes,
|
||||||
Usage: tokenUsageDTO{
|
Usage: tokenUsageDTO{
|
||||||
@@ -179,9 +174,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
StartTime: res.StartTime,
|
StartTime: res.StartTime,
|
||||||
EndTime: res.EndTime,
|
EndTime: res.EndTime,
|
||||||
DurationMS: res.Duration.Milliseconds(),
|
DurationMS: res.Duration.Milliseconds(),
|
||||||
ValidationMode: string(res.Validation.Mode),
|
|
||||||
ValidationStatus: string(res.Validation.Status),
|
|
||||||
RepairAttemptsUsed: res.Validation.RepairAttempts,
|
|
||||||
},
|
},
|
||||||
Execution: execution,
|
Execution: execution,
|
||||||
}
|
}
|
||||||
@@ -221,7 +213,6 @@ func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *
|
|||||||
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
|
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
|
||||||
return modelParamsDTO{
|
return modelParamsDTO{
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
BackendID: target.BackendID,
|
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
Temperature: target.Temperature,
|
Temperature: target.Temperature,
|
||||||
MaxTokens: target.MaxTokens,
|
MaxTokens: target.MaxTokens,
|
||||||
@@ -255,16 +246,16 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||||
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
|
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
|
||||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||||
|
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
||||||
|
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
||||||
|
case errors.Is(err, ErrFileTooLarge):
|
||||||
|
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
||||||
case errors.Is(err, promptkit.ErrPromptLoad):
|
case errors.Is(err, promptkit.ErrPromptLoad):
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
case errors.Is(err, promptkit.ErrProfileLoad):
|
case errors.Is(err, promptkit.ErrProfileLoad):
|
||||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
case errors.Is(err, promptkit.ErrInvalidRequest):
|
case errors.Is(err, promptkit.ErrInvalidRequest):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
|
||||||
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
|
||||||
case errors.Is(err, ErrFileTooLarge):
|
|
||||||
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
|
||||||
case errors.Is(err, promptkit.ErrArtifactLoad):
|
case errors.Is(err, promptkit.ErrArtifactLoad):
|
||||||
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
||||||
case errors.Is(err, promptkit.ErrPromptRender):
|
case errors.Is(err, promptkit.ErrPromptRender):
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -336,11 +337,13 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
|
|||||||
name, body string
|
name, body string
|
||||||
parent context.Context
|
parent context.Context
|
||||||
wantDeadline time.Time
|
wantDeadline time.Time
|
||||||
|
wantTimeout time.Duration
|
||||||
}{
|
}{
|
||||||
{name: "omitted inherits deadline", body: `{"prompt_id":"p"}`, parent: parent, wantDeadline: parentDeadline},
|
{name: "omitted inherits deadline", body: `{"prompt_id":"p"}`, parent: parent, wantDeadline: parentDeadline},
|
||||||
{name: "null inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: parent, wantDeadline: parentDeadline},
|
{name: "null inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: parent, wantDeadline: parentDeadline},
|
||||||
{name: "zero inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: parent, wantDeadline: parentDeadline},
|
{name: "zero inherits deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: parent, wantDeadline: parentDeadline},
|
||||||
{name: "positive preserves earlier deadline", body: `{"prompt_id":"p","run_timeout_seconds":10}`, parent: parent, wantDeadline: parentDeadline},
|
{name: "positive preserves earlier deadline", body: `{"prompt_id":"p","run_timeout_seconds":10}`, parent: parent, wantDeadline: parentDeadline, wantTimeout: 10 * time.Second},
|
||||||
|
{name: "positive adds operation budget", body: `{"prompt_id":"p","run_timeout_seconds":2}`, parent: context.Background(), wantTimeout: 2 * time.Second},
|
||||||
{name: "omitted adds no deadline", body: `{"prompt_id":"p"}`, parent: context.Background()},
|
{name: "omitted adds no deadline", body: `{"prompt_id":"p"}`, parent: context.Background()},
|
||||||
{name: "null adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: context.Background()},
|
{name: "null adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":null}`, parent: context.Background()},
|
||||||
{name: "zero adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: context.Background()},
|
{name: "zero adds no deadline", body: `{"prompt_id":"p","run_timeout_seconds":0}`, parent: context.Background()},
|
||||||
@@ -353,6 +356,9 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
|
|||||||
if ok != !tc.wantDeadline.IsZero() || (ok && !deadline.Equal(tc.wantDeadline)) {
|
if ok != !tc.wantDeadline.IsZero() || (ok && !deadline.Equal(tc.wantDeadline)) {
|
||||||
t.Fatalf("execution deadline: got %v (present=%t), want %v", deadline, ok, tc.wantDeadline)
|
t.Fatalf("execution deadline: got %v (present=%t), want %v", deadline, ok, tc.wantDeadline)
|
||||||
}
|
}
|
||||||
|
if req.OperationTimeout != tc.wantTimeout {
|
||||||
|
t.Fatalf("operation timeout = %v, want %v", req.OperationTimeout, tc.wantTimeout)
|
||||||
|
}
|
||||||
return &promptkit.RunResult{}, nil
|
return &promptkit.RunResult{}, nil
|
||||||
}))
|
}))
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -382,6 +388,22 @@ func TestHandlerRunTimeoutValidationAndContextMapping(t *testing.T) {
|
|||||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":10}`)))
|
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":10}`)))
|
||||||
assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
|
assertHTTPErrorCode(t, w, http.StatusBadGateway, "llm_failed")
|
||||||
|
|
||||||
|
expired, cancelExpired := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||||
|
defer cancelExpired()
|
||||||
|
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
|
return nil, fmt.Errorf("run failed: %w", ctx.Err())
|
||||||
|
}))
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p"}`)).WithContext(expired))
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
|
||||||
|
|
||||||
|
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
|
return nil, errors.Join(promptkit.ErrOperationTimeout, context.Canceled)
|
||||||
|
}))
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p"}`)).WithContext(canceled))
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
|
||||||
|
|
||||||
successContext, cancelSuccess := context.WithCancel(context.Background())
|
successContext, cancelSuccess := context.WithCancel(context.Background())
|
||||||
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
h = NewHandler(runnerFunc(func(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
cancelSuccess()
|
cancelSuccess()
|
||||||
@@ -447,7 +469,6 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
},
|
},
|
||||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
|
||||||
PromptHash: "phash",
|
PromptHash: "phash",
|
||||||
RenderedPromptHash: "rhash",
|
RenderedPromptHash: "rhash",
|
||||||
SelectedProfileID: "exec-default",
|
SelectedProfileID: "exec-default",
|
||||||
@@ -508,8 +529,10 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["prompt_id"] != "prompt-1" {
|
if metadata["prompt_id"] != "prompt-1" {
|
||||||
t.Fatalf("unexpected metadata.prompt_id: %#v", metadata["prompt_id"])
|
t.Fatalf("unexpected metadata.prompt_id: %#v", metadata["prompt_id"])
|
||||||
}
|
}
|
||||||
if metadata["prompt_version"] != "1.0.0" {
|
for _, field := range []string{"prompt_version", "model_name", "endpoint", "validation_mode", "validation_status", "repair_attempts_used"} {
|
||||||
t.Fatalf("unexpected metadata.prompt_version: %#v", metadata["prompt_version"])
|
if _, ok := metadata[field]; ok {
|
||||||
|
t.Fatalf("redundant metadata field %q: %#v", field, metadata)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if metadata["selected_profile_id"] != "exec-default" {
|
if metadata["selected_profile_id"] != "exec-default" {
|
||||||
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
t.Fatalf("unexpected metadata.selected_profile_id: %#v", metadata["selected_profile_id"])
|
||||||
@@ -517,9 +540,6 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["selected_backend_id"] != "local" {
|
if metadata["selected_backend_id"] != "local" {
|
||||||
t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"])
|
t.Fatalf("unexpected metadata.selected_backend_id: %#v", metadata["selected_backend_id"])
|
||||||
}
|
}
|
||||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
|
||||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
|
||||||
}
|
|
||||||
usage := metadata["usage"].(map[string]any)
|
usage := metadata["usage"].(map[string]any)
|
||||||
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
||||||
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
||||||
@@ -528,8 +548,8 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||||
}
|
}
|
||||||
modelParams := metadata["model_params"].(map[string]any)
|
modelParams := metadata["model_params"].(map[string]any)
|
||||||
if modelParams["backend_id"] != "local" {
|
if _, ok := modelParams["backend_id"]; ok || modelParams["model"] != "m1" || modelParams["endpoint"] != "http://llm/v1" {
|
||||||
t.Fatalf("expected model_params.backend_id=local, got %#v", modelParams["backend_id"])
|
t.Fatalf("unexpected model parameters: %#v", modelParams)
|
||||||
}
|
}
|
||||||
if modelParams["api_key_env"] != envName {
|
if modelParams["api_key_env"] != envName {
|
||||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||||
@@ -561,7 +581,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
func TestHandlerAllowsOmittedInputsAndPreservesQualifiedIDs(t *testing.T) {
|
||||||
r := &fakeRunner{result: &promptkit.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationNone, IsValid: true},
|
||||||
@@ -570,8 +590,8 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
|||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
for _, body := range []string{
|
for _, body := range []string{
|
||||||
`{"prompt_id":"prompt-1","prompt_version":"2"}`,
|
`{"prompt_id":"support/prompt-1","profile_id":"support/profile-1"}`,
|
||||||
`{"prompt_id":"prompt-1","prompt_version":"2","inputs":{}}`,
|
`{"prompt_id":"support/prompt-1","profile_id":"support/profile-1","inputs":{}}`,
|
||||||
} {
|
} {
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -580,8 +600,8 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
|||||||
if w.Code != http.StatusOK {
|
if w.Code != http.StatusOK {
|
||||||
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
}
|
}
|
||||||
if r.last.PromptVersion != "2" {
|
if r.last.PromptID != "support/prompt-1" || r.last.ProfileID != "support/profile-1" {
|
||||||
t.Fatalf("expected prompt version to be mapped, got %q", r.last.PromptVersion)
|
t.Fatalf("qualified IDs changed: %+v", r.last)
|
||||||
}
|
}
|
||||||
if r.last.Inputs != nil {
|
if r.last.Inputs != nil {
|
||||||
t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs)
|
t.Fatalf("expected omitted or empty inputs to remain nil, got %#v", r.last.Inputs)
|
||||||
@@ -589,6 +609,16 @@ func TestHandlerAllowsOmittedInputsAndMapsPromptVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerRejectsRemovedPromptVersion(t *testing.T) {
|
||||||
|
runner := &fakeRunner{result: &promptkit.RunResult{}}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
NewHandler(runner).ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","prompt_version":"1"}`)))
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
||||||
|
if runner.last.PromptID != "" {
|
||||||
|
t.Fatal("runner invoked for removed selector")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
func TestHandlerDelegatesDefinitionInputRequirements(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -786,7 +816,6 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|||||||
r := &fakeRunner{result: &promptkit.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
|
||||||
SelectedProfileID: "prompt-default",
|
SelectedProfileID: "prompt-default",
|
||||||
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
@@ -1191,9 +1220,13 @@ func TestHandlerDeadlineCancelsBuiltInRetryWait(t *testing.T) {
|
|||||||
}))
|
}))
|
||||||
defer provider.Close()
|
defer provider.Close()
|
||||||
h := newRetryHandler(t, provider.URL+"/v1", intPointer(1), 0, nil)
|
h := newRetryHandler(t, provider.URL+"/v1", intPointer(1), 0, nil)
|
||||||
|
failures := observeRunnerFailures(h)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
|
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
|
||||||
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
|
assertHTTPErrorCode(t, w, http.StatusGatewayTimeout, "deadline_exceeded")
|
||||||
|
if err := <-failures; !errors.Is(err, promptkit.ErrOperationTimeout) {
|
||||||
|
t.Fatalf("engine did not attribute retry wait to operation timeout: %v", err)
|
||||||
|
}
|
||||||
if atomic.LoadInt32(&hits) != 1 {
|
if atomic.LoadInt32(&hits) != 1 {
|
||||||
t.Fatalf("expected one provider call before canceled retry wait, got %d", atomic.LoadInt32(&hits))
|
t.Fatalf("expected one provider call before canceled retry wait, got %d", atomic.LoadInt32(&hits))
|
||||||
}
|
}
|
||||||
@@ -1221,6 +1254,7 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
|
|||||||
}()
|
}()
|
||||||
queue := 1
|
queue := 1
|
||||||
h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 1, &queue)
|
h := newRetryHandler(t, provider.URL+"/v1", intPointer(0), 1, &queue)
|
||||||
|
failures := observeRunnerFailures(h)
|
||||||
first := httptest.NewRecorder()
|
first := httptest.NewRecorder()
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
@@ -1235,6 +1269,9 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
|
|||||||
waiting := httptest.NewRecorder()
|
waiting := httptest.NewRecorder()
|
||||||
h.ServeHTTP(waiting, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
|
h.ServeHTTP(waiting, httptest.NewRequest(http.MethodPost, "/v1/runs", strings.NewReader(`{"prompt_id":"p","run_timeout_seconds":1}`)))
|
||||||
assertHTTPErrorCode(t, waiting, http.StatusGatewayTimeout, "deadline_exceeded")
|
assertHTTPErrorCode(t, waiting, http.StatusGatewayTimeout, "deadline_exceeded")
|
||||||
|
if err := <-failures; !errors.Is(err, promptkit.ErrOperationTimeout) {
|
||||||
|
t.Fatalf("engine did not attribute capacity wait to operation timeout: %v", err)
|
||||||
|
}
|
||||||
if atomic.LoadInt32(&hits) != 1 {
|
if atomic.LoadInt32(&hits) != 1 {
|
||||||
t.Fatalf("capacity waiter reached provider: hits=%d", atomic.LoadInt32(&hits))
|
t.Fatalf("capacity waiter reached provider: hits=%d", atomic.LoadInt32(&hits))
|
||||||
}
|
}
|
||||||
@@ -1254,13 +1291,25 @@ func TestHandlerDeadlineCancelsCapacityWaitAndReleasesAdmission(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func observeRunnerFailures(h *Handler) <-chan error {
|
||||||
|
failures := make(chan error, 1)
|
||||||
|
runner := h.runner
|
||||||
|
h.runner = runnerFunc(func(ctx context.Context, request promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
|
result, err := runner.Run(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
failures <- err
|
||||||
|
}
|
||||||
|
return result, err
|
||||||
|
})
|
||||||
|
return failures
|
||||||
|
}
|
||||||
|
|
||||||
func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency int, queue *int) *Handler {
|
func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency int, queue *int) *Handler {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
promptDir := t.TempDir()
|
promptDir := t.TempDir()
|
||||||
profileDir := t.TempDir()
|
profileDir := t.TempDir()
|
||||||
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`prompts:
|
||||||
p:
|
- id: p
|
||||||
"1":
|
|
||||||
default_profile: limited
|
default_profile: limited
|
||||||
messages:
|
messages:
|
||||||
- role: user
|
- role: user
|
||||||
@@ -1271,10 +1320,10 @@ func newRetryHandler(t *testing.T, endpoint string, retries *int, concurrency in
|
|||||||
`), 0o644); err != nil {
|
`), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n limited:\n backend: limited\n model: test\n"), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte("profiles:\n - id: limited\n backend: limited\n model: test\n"), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
backend := promptkit.BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrency, QueueCapacity: queue}
|
backend := promptkit.BackendConfig{ID: "limited", Endpoint: &endpoint, ConcurrencyLimit: &concurrency, QueueCapacity: queue}
|
||||||
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, retries, &backend)
|
engine := newHandlerEngineFromDirectories(t, promptDir, profileDir, retries, &backend)
|
||||||
return NewHandler(engine)
|
return NewHandler(engine)
|
||||||
}
|
}
|
||||||
@@ -1627,6 +1676,7 @@ func TestHandlerPublicErrorMapping(t *testing.T) {
|
|||||||
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
||||||
|
{name: "file too large with broad category", err: errors.Join(ErrFileTooLarge, promptkit.ErrInvalidRequest, promptkit.ErrArtifactLoad), status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
||||||
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||||
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||||
{name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"},
|
{name: "capacity", err: &promptkit.CapacityError{BackendID: "private-backend"}, status: http.StatusServiceUnavailable, code: "capacity_exceeded", message: "model backend capacity is exhausted", avoidCause: "private-backend"},
|
||||||
@@ -1829,7 +1879,7 @@ func newHandlerEngineFromDirectories(t *testing.T, promptDir, profileDir string,
|
|||||||
}
|
}
|
||||||
underlays := []configkit.Underlay{{Inputs: promptInputs}, {Inputs: profileInputs}}
|
underlays := []configkit.Underlay{{Inputs: promptInputs}, {Inputs: profileInputs}}
|
||||||
if backend != nil {
|
if backend != nil {
|
||||||
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource("test backend", map[string]promptkit.BackendConfig{"limited": *backend})}}})
|
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource("test backend", []promptkit.BackendConfig{*backend})}}})
|
||||||
}
|
}
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
|||||||
Reference in New Issue
Block a user