Document invocation observation and cost reporting
This commit is contained in:
@@ -1,12 +1,29 @@
|
|||||||
package promptkit_test
|
package promptkit_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"math"
|
"math"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDuplicateContextAndRequestGroupEnrollment(t *testing.T) {
|
||||||
|
charge := .5
|
||||||
|
engine := costEngine(t, false, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
return &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil
|
||||||
|
})))
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
ctx := promptkit.WithExecutionObserver(context.Background(), group.Observe)
|
||||||
|
if _, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: group.Observe}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := group.Snapshot()
|
||||||
|
if len(got.Members) != 1 || got.Attempts != 1 || got.CostUSD.Value != .5 {
|
||||||
|
t.Fatalf("summary=%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
|
func TestCacheHitRatioUsesCompleteConsistentCounts(t *testing.T) {
|
||||||
known := func(v int) promptkit.ObservedTokenCount {
|
known := func(v int) promptkit.ObservedTokenCount {
|
||||||
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
return promptkit.ObservedTokenCount{Value: v, Known: true, Complete: true}
|
||||||
|
|||||||
28
cost.go
28
cost.go
@@ -6,19 +6,31 @@ import "go.maximumdirect.net/promptkit/internal/domain"
|
|||||||
type CostSource string
|
type CostSource string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
CostSourceNone CostSource = ""
|
// CostSourceNone denotes no known sourced contribution.
|
||||||
CostSourceProvider CostSource = "provider"
|
CostSourceNone CostSource = ""
|
||||||
|
// CostSourceProvider denotes a reported provider or injected-client amount.
|
||||||
|
CostSourceProvider CostSource = "provider"
|
||||||
|
// CostSourceConfiguredZero denotes an explicitly configured zero fallback.
|
||||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||||
CostSourceMixed CostSource = "mixed"
|
// CostSourceMixed denotes both provider and configured-zero contributions.
|
||||||
|
CostSourceMixed CostSource = "mixed"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ObservedCost reports an available USD subtotal and its coverage and source.
|
// ObservedCost reports an available USD subtotal and its coverage and source.
|
||||||
// Complete describes coverage of dispatched attempts, not billing verification.
|
// Known means at least one contribution is available. Complete means all
|
||||||
|
// included dispatched attempts have an amount under the selected policy; it
|
||||||
|
// does not verify billing. A completed run with no dispatched attempts, or an
|
||||||
|
// empty group, has known complete zero with no source. A group with members but
|
||||||
|
// no terminals has unknown cost. Amounts are unrounded binary floating point.
|
||||||
type ObservedCost struct {
|
type ObservedCost struct {
|
||||||
Value float64 `json:"value"`
|
// Value is the known USD subtotal.
|
||||||
Known bool `json:"known"`
|
Value float64 `json:"value"`
|
||||||
Complete bool `json:"complete"`
|
// Known reports whether a subtotal is available.
|
||||||
Source CostSource `json:"source"`
|
Known bool `json:"known"`
|
||||||
|
// Complete reports whether all included dispatched attempts are covered.
|
||||||
|
Complete bool `json:"complete"`
|
||||||
|
// Source summarizes the known contributions.
|
||||||
|
Source CostSource `json:"source"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func fromDomainCost(value domain.ObservedCost) ObservedCost {
|
func fromDomainCost(value domain.ObservedCost) ObservedCost {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -62,6 +63,7 @@ func TestOrdinaryRunCostMatchesTerminalWithOrWithoutObserver(t *testing.T) {
|
|||||||
|
|
||||||
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
|
func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
|
||||||
charge := .125
|
charge := .125
|
||||||
|
invalid := math.Inf(1)
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
response *promptkit.GenerateResponse
|
response *promptkit.GenerateResponse
|
||||||
@@ -70,6 +72,7 @@ func TestOrdinaryRunCostFailureAndFallback(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
|
{"reported takes precedence", &promptkit.GenerateResponse{Content: "ok", CostUSD: &charge}, nil, promptkit.ObservedCost{Value: .125, Known: true, Complete: true, Source: promptkit.CostSourceProvider}},
|
||||||
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
{"failed dispatch fallback", nil, errors.New("failure"), promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
||||||
|
{"nonfinite report falls back", &promptkit.GenerateResponse{Content: "ok", CostUSD: &invalid}, nil, promptkit.ObservedCost{Known: true, Complete: true, Source: promptkit.CostSourceConfiguredZero}},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
engine := costEngine(t, true, nil, promptkit.WithLLMClient(observationClientFunc(func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
|||||||
@@ -728,6 +728,40 @@ counts, but its zero counts remain unknown to observations; supply explicit
|
|||||||
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
|
presence when measured zeros matter. Existing `RunResult.Usage` sums returned
|
||||||
values independently of observation presence.
|
values independently of observation presence.
|
||||||
|
|
||||||
|
For an application-owned invocation spanning several calls or engines, attach
|
||||||
|
an observer once to a context and propagate it to those calls:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var group promptkit.ExecutionGroup
|
||||||
|
ctx = promptkit.WithExecutionObserver(ctx, group.Observe)
|
||||||
|
result, err := engine.Run(ctx, request)
|
||||||
|
// After joining all application-owned work, inspect group.Snapshot(), even on error.
|
||||||
|
```
|
||||||
|
|
||||||
|
Explicit request observers run first, followed by contextual observers in
|
||||||
|
registration order. Coordinated continuations pick up the context through
|
||||||
|
`Run`; prepared handles retain the request observer but use the execution
|
||||||
|
context's observers, not the preparation context's. Each nested call has its
|
||||||
|
own run ID and accounting. Duplicate registration can deliver duplicate events;
|
||||||
|
groups count each run's first terminal once.
|
||||||
|
|
||||||
|
Reasoning tokens are a subset of completion tokens; cache reads and writes are
|
||||||
|
input details, not extra tokens. `ObservedTokenUsage.CacheHitRatio()` derives
|
||||||
|
the cache-read fraction only when input and read totals are known, complete,
|
||||||
|
positive-denominator, and consistent. It divides aggregate counts rather than
|
||||||
|
averaging per-run ratios and is independent of cache writes.
|
||||||
|
|
||||||
|
`RunResult.CostUSD` is available even without an observer; attempt and terminal
|
||||||
|
events and group snapshots also report cost. `Known` denotes an available
|
||||||
|
subtotal, `Complete` covers dispatched attempts, and `Source` distinguishes
|
||||||
|
provider reports, an explicitly configured zero fallback, and mixed sources.
|
||||||
|
Reported cost, including zero, takes precedence over fallback. Unknown failed
|
||||||
|
attempts make a subtotal incomplete; configured zero may cover them. Terminal
|
||||||
|
events can report cost when a failed operation returns no result. Neither
|
||||||
|
complete reporting nor provider attribution verifies an invoice; consumers
|
||||||
|
choose presentation precision. See [backend definitions](../formats.md#backend-definitions)
|
||||||
|
and the [integration contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||||
|
|
||||||
Callbacks run synchronously, in order for one operation, and can overlap for
|
Callbacks run synchronously, in order for one operation, and can overlap for
|
||||||
different operations. Synchronize shared callback state and avoid blocking on
|
different operations. Synchronize shared callback state and avoid blocking on
|
||||||
work that needs the same run's admission slot. Callback panics propagate;
|
work that needs the same run's admission slot. Callback panics propagate;
|
||||||
@@ -879,15 +913,15 @@ outcome, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
|||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
`assess` is application code and explicitly selects its observer on any nested
|
`assess` is application code and can select its observer explicitly or inherit
|
||||||
run. A completed turn releases its admission lease before the callback, so that
|
one from the context. A completed turn releases its admission lease before the callback, so that
|
||||||
assessment may use the same backend. `stopped` and `exhausted` describe control
|
assessment may use the same backend. `stopped` and `exhausted` describe control
|
||||||
flow, not content acceptance. Cancellation before callback delivery can return one
|
flow, not content acceptance. Cancellation before callback delivery can return one
|
||||||
completed response in `Undelivered`; errors retain their ordinary identities.
|
completed response in `Undelivered`; errors retain their ordinary identities.
|
||||||
|
|
||||||
### Collect Selected Execution Facts
|
### Collect Selected Execution Facts
|
||||||
|
|
||||||
Groups are independent of continuation and enrollment is explicit:
|
Groups are independent of continuation and can be enrolled explicitly:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
var group, validatorGroup promptkit.ExecutionGroup
|
var group, validatorGroup promptkit.ExecutionGroup
|
||||||
@@ -904,7 +938,7 @@ _, _ = summary, validatorSummary
|
|||||||
```
|
```
|
||||||
|
|
||||||
Only the first terminal event per run contributes counts and usage; coverage of
|
Only the first terminal event per run contributes counts and usage; coverage of
|
||||||
currently observed members is separate from token-usage completeness. Empty
|
currently observed members is separate from token and cost completeness. Empty
|
||||||
groups report complete, known zero totals, while members without terminals do
|
groups report complete, known zero totals, while members without terminals do
|
||||||
not invent totals. Overlapping groups are not generally additive. See the
|
not invent totals. Overlapping groups are not generally additive. See the
|
||||||
[offline example](../../examples/go-library/observe/main.go) for complete setup.
|
[offline example](../../examples/go-library/observe/main.go) for complete setup.
|
||||||
|
|||||||
@@ -223,7 +223,7 @@ Backend definitions are supplied through caller-selected configuration
|
|||||||
sources. The conventional backend IDs are names only; PromptKit does not
|
sources. The conventional backend IDs are names only; PromptKit does not
|
||||||
install definitions for them.
|
install definitions for them.
|
||||||
|
|
||||||
A backend definition may have `endpoint`, `api_key_env`,
|
A backend definition may have `provider`, `zero_cost`, `endpoint`, `api_key_env`,
|
||||||
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
`concurrency_limit`, `queue_capacity`, `extra_params`, and `smoke_tests`. The
|
||||||
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
final endpoint is required and must be an absolute HTTP(S) URL with a host, no
|
||||||
user information, query, or fragment. An omitted or null API-key environment
|
user information, query, or fragment. An omitted or null API-key environment
|
||||||
@@ -237,6 +237,17 @@ unrecognized backend field is application data and is not automatically redacted
|
|||||||
by Promptkit. Applications own sensitivity declarations and disclosure policy
|
by Promptkit. Applications own sensitivity declarations and disclosure policy
|
||||||
for their extension fields.
|
for their extension fields.
|
||||||
|
|
||||||
|
`provider` selects response and diagnostic conventions independently of the
|
||||||
|
backend ID, endpoint, or model. Its final value is trimmed and lowercased;
|
||||||
|
omitted, null, or blank selects `openai-compatible` at runtime. Supported
|
||||||
|
nonblank values are `openai-compatible`, `openrouter`, and `vllm`. `zero_cost`
|
||||||
|
is an optional Boolean: true supplies a zero-cost fallback when a dispatched
|
||||||
|
attempt has no usable reported cost; omitted, null, and false disable it.
|
||||||
|
Neither setting is sent to the provider. They follow normal layering and
|
||||||
|
inheritance, and endpoint overrides retain the selected backend's policy.
|
||||||
|
Resource inspection shows declared normalized values without inventing runtime
|
||||||
|
defaults; configuration inspection retains source history.
|
||||||
|
|
||||||
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
`smoke_tests` is optional backend metadata for explicit diagnostic operations.
|
||||||
It may contain `default_profile`, `reachability`, and `authentication`:
|
It may contain `default_profile`, `reachability`, and `authentication`:
|
||||||
|
|
||||||
@@ -274,6 +285,10 @@ Without a selected adapter, authentication diagnostics report an unsupported
|
|||||||
selection. An endpoint-only authentication
|
selection. An endpoint-only authentication
|
||||||
block overrides the destination of a provider default; `kind: none` suppresses
|
block overrides the destination of a provider default; `kind: none` suppresses
|
||||||
that default.
|
that default.
|
||||||
|
Previously, a supplied authentication object required `kind`. Endpoint-only
|
||||||
|
objects now use a provider default where available; clearing the object
|
||||||
|
restores that default rather than suppressing it. Use `kind: none` to suppress
|
||||||
|
authentication testing.
|
||||||
These values are configuration only: engine construction does not contact a
|
These values are configuration only: engine construction does not contact a
|
||||||
backend.
|
backend.
|
||||||
|
|
||||||
@@ -364,7 +379,7 @@ does not disable the caller context or HTTP transport timeout.
|
|||||||
|
|
||||||
### Bundled OpenRouter Configuration
|
### Bundled OpenRouter Configuration
|
||||||
|
|
||||||
The default bundle installs root backend `openrouter`, endpoint
|
The default bundle installs root backend `openrouter`, provider `openrouter`, endpoint
|
||||||
`https://openrouter.ai/api/v1`, optional credential environment name
|
`https://openrouter.ai/api/v1`, optional credential environment name
|
||||||
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
`OPENROUTER_API_KEY`, and smoke metadata selecting `gpt-luna-latest` with the
|
||||||
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
`openrouter_key` authentication adapter. Existing diagnostic endpoint fallback
|
||||||
|
|||||||
@@ -173,6 +173,10 @@ otherwise usable content. Existing common count decoding remains strict.
|
|||||||
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
|
OpenRouter's `usage.cost` is an optional USD amount; malformed or non-finite
|
||||||
amounts are ignored without invalidating content. The generic and vLLM
|
amounts are ignored without invalidating content. The generic and vLLM
|
||||||
providers do not interpret that field.
|
providers do not interpret that field.
|
||||||
|
vLLM prompt-token details depend on server configuration, including
|
||||||
|
`--enable-prompt-tokens-details`; reasoning details depend on a configured
|
||||||
|
reasoning parser. Servers without these details remain usable. vLLM cache
|
||||||
|
writes describe local prefix-cache creation, not a billable write charge.
|
||||||
|
|
||||||
Each common token field retains whether the provider reported a
|
Each common token field retains whether the provider reported a
|
||||||
non-null integer, including zero. Omitted and `null` fields are unknown for
|
non-null integer, including zero. Omitted and `null` fields are unknown for
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ other backend IDs.
|
|||||||
The coordinator does not retry injected clients; it normalizes a nil successful
|
The coordinator does not retry injected clients; it normalizes a nil successful
|
||||||
response as malformed generation. The built-in HTTP client remains responsible
|
response as malformed generation. The built-in HTTP client remains responsible
|
||||||
for provider transport behavior. The runner, rather than the
|
for provider transport behavior. The runner, rather than the
|
||||||
capacity layer, sums all five usage fields from the initial response and every
|
capacity layer, sums usage fields from the initial response and every
|
||||||
completed repair response into the successful run result.
|
completed repair response into the successful run result.
|
||||||
|
|
||||||
An invocation-local observer receives attempt events after active permit
|
An invocation-local observer receives attempt events after active permit
|
||||||
|
|||||||
@@ -35,6 +35,12 @@ unknown usage. Transport, timeout, malformed, and noneligible status failures
|
|||||||
stop the sequence without retry exhaustion. Context is checked before each
|
stop the sequence without retry exhaustion. Context is checked before each
|
||||||
wait, acquisition, and dispatch.
|
wait, acquisition, and dispatch.
|
||||||
|
|
||||||
|
At every dispatched completion the coordinator resolves any reported cost
|
||||||
|
against the frozen backend zero-cost policy and adds it once to the run's
|
||||||
|
[cost accumulator](observation.md), independently of observer selection.
|
||||||
|
Injected responses may retain reported cost alongside an error. Undispatched
|
||||||
|
acquisitions and retry waits contribute no cost.
|
||||||
|
|
||||||
The public client adapter translates explicitly supplied public transport and
|
The public client adapter translates explicitly supplied public transport and
|
||||||
malformed-response identities into internal categories while retaining the
|
malformed-response identities into internal categories while retaining the
|
||||||
original error chain. This keeps injected attempt classifications consistent
|
original error chain. This keeps injected attempt classifications consistent
|
||||||
|
|||||||
@@ -142,6 +142,14 @@ known. Strictly decoded usage can still be observed if later choice/content
|
|||||||
checks fail; invalid framing or a non-success error body contributes none.
|
checks fail; invalid framing or a non-success error body contributes none.
|
||||||
Existing successful-result accounting remains value-based.
|
Existing successful-result accounting remains value-based.
|
||||||
|
|
||||||
|
The captured provider selects additional response details and authentication
|
||||||
|
diagnostic defaults. Optional new reasoning, cache-write, and OpenRouter cost
|
||||||
|
fields are decoded independently, without relaxing common-field or framing
|
||||||
|
validation. Cost remains an optional raw amount at this boundary; shared
|
||||||
|
accounting applies configured-zero fallback after dispatch. The integration
|
||||||
|
contract specifies the [wire mappings](../integrations/openai-compatible-chat.md#response-handling)
|
||||||
|
and [probe recognition](../integrations/openai-compatible-chat.md#explicit-smoke-diagnostics).
|
||||||
|
|
||||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||||
only normalized detail from the bounded recognized envelope. It retains
|
only normalized detail from the bounded recognized envelope. It retains
|
||||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ contributor workflow and validation.
|
|||||||
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
| `internal/domain` | Defines internal framework values for requests, immutable conversation exchanges, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, validation, and source-neutral smoke diagnostics, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, output contracts, and resource-identity syntax and qualification. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [exchange storage](../../internal/domain/exchange.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
|
||||||
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
| `internal/engine` | Assembles an already captured configuration into shared capacity, rendering, validation, generation, repair, runner, and smoke-test collaborators. Immutable namespace views share those collaborators and qualify selections without rewriting captured references. Built-in engines provide the diagnostic HTTP collaborator; injected-client engines do not. It delegates resource inspection and listing to the capture. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#typed-configuration-loading), [internal runner](runner.md) |
|
||||||
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, ordinary attempt observations, and invocation-local evidence for diagnostics. | [Internal generation coordination](generation.md) |
|
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded retries, per-attempt cost, ordinary observations, and separate diagnostic evidence. | [Internal generation coordination](generation.md) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible provider values, and detaches previously accepted values without repeating validation. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
@@ -30,8 +30,8 @@ contributor workflow and validation.
|
|||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Creates operation-local validation plans from inline schema documents or bound filesystem resources and validates basic, JSON, and JSON Schema output. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded response decoding, authentication, deadline handling, per-invocation HTTP evidence and built-in diagnostic probes, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible requests, provider-specific response interpretation and authentication probes, bounded decoding, deadlines, and diagnostic evidence. | [Internal model client](llm.md) |
|
||||||
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, synchronous metadata event delivery and composition, and explicit terminal-based group aggregation; smoke diagnostics keep their private evidence outside ordinary observer events. | [Internal execution observation](observation.md) |
|
| `internal/observation` | Owns invocation-local run IDs, presence-aware token and cost accounting, contextual observer selection, synchronous event delivery, and terminal-based group aggregation; smoke diagnostics retain separate evidence. | [Internal execution observation](observation.md) |
|
||||||
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair, advances a single conversation branch, and runs explicit smoke diagnostics with private cases, shared generation/capacity, and optional built-in HTTP probes. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The internal engine assembles these internal components without exposing their
|
The internal engine assembles these internal components without exposing their
|
||||||
|
|||||||
@@ -216,10 +216,16 @@ ordinary `Run` reuses its run ID and carries it to the generation coordinator
|
|||||||
through context. Prompt and profile identities are enriched only after their
|
through context. Prompt and profile identities are enriched only after their
|
||||||
respective resolutions succeed, before admission. The facade starts observation
|
respective resolutions succeed, before admission. The facade starts observation
|
||||||
before request conversion and emits its terminal event after public error
|
before request conversion and emits its terminal event after public error
|
||||||
mapping and after the runner's deferred admission release. Preparation and
|
mapping and after the runner's deferred admission release. Preparation emits no
|
||||||
prepared execution follows the same terminal mapping after a successful claim;
|
events; prepared execution follows the same terminal mapping after a successful claim;
|
||||||
invalid or discarded handles do not acquire an observer.
|
invalid or discarded handles do not acquire an observer.
|
||||||
|
|
||||||
|
Ordinary runs establish fresh private cost state before observation; a claimed
|
||||||
|
prepared execution establishes its own state after claim. Contextual observers
|
||||||
|
remain available at public boundaries, while enclosing recorder and cost state
|
||||||
|
are masked. Prepared handles retain the request observer and frozen provider
|
||||||
|
policy, not preparation-context selection or an accumulator.
|
||||||
|
|
||||||
The facade establishes an optional operation scope for ordinary runs before
|
The facade establishes an optional operation scope for ordinary runs before
|
||||||
observation and preparation. A claimed prepared execution establishes its own
|
observation and preparation. A claimed prepared execution establishes its own
|
||||||
scope from the retained duration; preparation and idle time do not consume it.
|
scope from the retained duration; preparation and idle time do not consume it.
|
||||||
@@ -273,11 +279,13 @@ run identifier, and UTC timing. The same effective session reaches initial
|
|||||||
generation and any repair attempt through the rendered prompt. The same
|
generation and any repair attempt through the rendered prompt. The same
|
||||||
effective target and presence metadata, including backend identity and direct
|
effective target and presence metadata, including backend identity and direct
|
||||||
credential during execution, reaches generation and every repair attempt.
|
credential during execution, reaches generation and every repair attempt.
|
||||||
Result usage is the field-wise sum of all five usage values from the initial
|
Result usage is the field-wise sum of usage values from the initial
|
||||||
response and every completed repair response. Final raw output, artifact, and
|
response and every completed repair response. Final raw output, artifact, and
|
||||||
validation state still come from the last candidate. A repair error returns no
|
validation state still come from the last candidate. A repair error returns no
|
||||||
partial run result or partial result usage, although the invocation-local
|
partial run result or partial result usage, although the invocation-local
|
||||||
observer retains completed-attempt usage and reports it in the terminal event.
|
observer retains completed-attempt usage and reports it in the terminal event.
|
||||||
|
Result cost instead snapshots the shared accumulator across all dispatched
|
||||||
|
attempts, including retries and repairs, whether or not observation is enabled.
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
|
|||||||
@@ -417,6 +417,8 @@ implementation has reached its final buildable state.
|
|||||||
|
|
||||||
## Stage 15 — Canonical Documentation And Final Verification
|
## Stage 15 — Canonical Documentation And Final Verification
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
- Finalize GoDoc for helpers, backend fields, reasoning counts, cost values and
|
- Finalize GoDoc for helpers, backend fields, reasoning counts, cost values and
|
||||||
sources, client ownership, and observer/prepared lifetimes. Update stale
|
sources, client ownership, and observer/prepared lifetimes. Update stale
|
||||||
authentication Kind requirements in shared settings and public docs.
|
authentication Kind requirements in shared settings and public docs.
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# Invocation-Scoped Execution Observation
|
# Invocation-Scoped Execution Observation
|
||||||
|
|
||||||
Status: accepted scope; implementation pending.
|
Status: accepted scope; implementation complete. Current behavior is documented
|
||||||
|
in the linked canonical contracts; roadmap retirement is separate housekeeping.
|
||||||
|
|
||||||
## Purpose And Scope
|
## Purpose And Scope
|
||||||
|
|
||||||
|
|||||||
@@ -535,7 +535,8 @@ func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.Pr
|
|||||||
// returns no handle on error. A nil Engine returns an error matching
|
// returns no handle on error. A nil Engine returns an error matching
|
||||||
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
// ErrInvalidConfig. The effective transient retry budget is frozen privately
|
||||||
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
// for RunPrepared and is absent from Details. A supplied Observer is retained
|
||||||
// privately for RunPrepared; preparation itself emits no events.
|
// privately for RunPrepared; preparation itself emits no events and does not
|
||||||
|
// retain contextual observers.
|
||||||
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
|
// RunRequest.OperationTimeout and RunRequest.OnPrepared are retained without
|
||||||
// starting a timer or invoking the notification during preparation.
|
// starting a timer or invoking the notification during preparation.
|
||||||
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||||
@@ -692,8 +693,9 @@ func (e *Engine) run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
|
// error. RunPrepared uses the retry choice frozen during PrepareExecution. An
|
||||||
// operational error returns no partial RunResult. The retained Observer and
|
// operational error returns no partial RunResult. The retained Observer and
|
||||||
// execution-context observers begin only after a successful atomic claim, with
|
// execution-context observers begin only after a successful atomic claim, with
|
||||||
// identities from the frozen snapshot, and receive a terminal event on normal return after public error
|
// identities from the frozen snapshot, and receive a terminal event on normal
|
||||||
// mapping. Invalid, foreign, discarded, or reused handles emit no events.
|
// return after public error mapping. Invalid, foreign, discarded, or reused
|
||||||
|
// handles emit no events.
|
||||||
// Callbacks run outside the handle and capacity locks; panics propagate and
|
// Callbacks run outside the handle and capacity locks; panics propagate and
|
||||||
// do not promise a terminal event.
|
// do not promise a terminal event.
|
||||||
// The retained RunRequest.OperationTimeout starts after claim; the retained
|
// The retained RunRequest.OperationTimeout starts after claim; the retained
|
||||||
|
|||||||
@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
|||||||
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
||||||
body, _ := json.Marshal(map[string]any{
|
body, _ := json.Marshal(map[string]any{
|
||||||
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
||||||
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
|
"usage": map[string]any{
|
||||||
|
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
|
||||||
|
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
|
||||||
|
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
return httpResponse(http.StatusOK, string(body))
|
return httpResponse(http.StatusOK, string(body))
|
||||||
}
|
}
|
||||||
@@ -80,6 +84,10 @@ type summary struct {
|
|||||||
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||||
GroupMembers int `json:"group_members"`
|
GroupMembers int `json:"group_members"`
|
||||||
GroupAttempts uint64 `json:"group_attempts"`
|
GroupAttempts uint64 `json:"group_attempts"`
|
||||||
|
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||||
|
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||||
|
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||||
|
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||||
PreparedMessages int `json:"prepared_messages"`
|
PreparedMessages int `json:"prepared_messages"`
|
||||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||||
}
|
}
|
||||||
@@ -97,6 +105,8 @@ func run() (summary, error) {
|
|||||||
filesystem := configkit.NewFilesystem("example schemas", assets)
|
filesystem := configkit.NewFilesystem("example schemas", assets)
|
||||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||||
backend.ID = backendID
|
backend.ID = backendID
|
||||||
|
provider, zeroCost := "openrouter", true
|
||||||
|
backend.Provider, backend.ZeroCost = &provider, &zeroCost
|
||||||
inputs = append(inputs,
|
inputs = append(inputs,
|
||||||
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
||||||
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||||
@@ -117,8 +127,9 @@ func run() (summary, error) {
|
|||||||
|
|
||||||
var proposalLog eventLog
|
var proposalLog eventLog
|
||||||
var executions promptkit.ExecutionGroup
|
var executions promptkit.ExecutionGroup
|
||||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
|
||||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
proposal, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return summary{}, err
|
return summary{}, err
|
||||||
@@ -133,8 +144,8 @@ func run() (summary, error) {
|
|||||||
var assessmentLog eventLog
|
var assessmentLog eventLog
|
||||||
var preparedDetails promptkit.PreparedRun
|
var preparedDetails promptkit.PreparedRun
|
||||||
decision := "rejected"
|
decision := "rejected"
|
||||||
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||||
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
Start: proposal.Exchange, AdditionalTurns: 1,
|
||||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||||
if turn.Result == nil {
|
if turn.Result == nil {
|
||||||
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
||||||
@@ -142,7 +153,7 @@ func run() (summary, error) {
|
|||||||
if !turn.Result.Validation.IsValid {
|
if !turn.Result.Validation.IsValid {
|
||||||
return nil, fmt.Errorf("follow-up failed structural validation")
|
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||||
}
|
}
|
||||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -181,6 +192,8 @@ func run() (summary, error) {
|
|||||||
return summary{}, err
|
return summary{}, err
|
||||||
}
|
}
|
||||||
group := executions.Snapshot()
|
group := executions.Snapshot()
|
||||||
|
_, ratioAvailable := group.Usage.CacheHitRatio()
|
||||||
|
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
|
||||||
return summary{
|
return summary{
|
||||||
ProposalAttempts: proposalTerminal.Attempts,
|
ProposalAttempts: proposalTerminal.Attempts,
|
||||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||||
@@ -192,6 +205,10 @@ func run() (summary, error) {
|
|||||||
ExhaustedOutcome: exhausted.Outcome,
|
ExhaustedOutcome: exhausted.Outcome,
|
||||||
GroupMembers: len(group.Members),
|
GroupMembers: len(group.Members),
|
||||||
GroupAttempts: group.Attempts,
|
GroupAttempts: group.Attempts,
|
||||||
|
GroupCost: group.CostUSD,
|
||||||
|
GroupReasoning: group.Usage.ReasoningTokens,
|
||||||
|
GroupCacheRatioAvailable: ratioAvailable,
|
||||||
|
AssessmentCacheRatio: assessmentRatio,
|
||||||
PreparedMessages: len(preparedDetails.Messages),
|
PreparedMessages: len(preparedDetails.Messages),
|
||||||
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
||||||
}, nil
|
}, nil
|
||||||
|
|||||||
@@ -3,7 +3,10 @@ package main
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"math"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestObserveExample(t *testing.T) {
|
func TestObserveExample(t *testing.T) {
|
||||||
@@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
var got struct {
|
var got struct {
|
||||||
ProposalAttempts *int `json:"proposal_attempts"`
|
ProposalAttempts *int `json:"proposal_attempts"`
|
||||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||||
Outcome string `json:"assessment_outcome"`
|
Outcome string `json:"assessment_outcome"`
|
||||||
CoordinationOutcome string `json:"coordination_outcome"`
|
CoordinationOutcome string `json:"coordination_outcome"`
|
||||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||||
GroupMembers int `json:"group_members"`
|
GroupMembers int `json:"group_members"`
|
||||||
GroupAttempts int `json:"group_attempts"`
|
GroupAttempts int `json:"group_attempts"`
|
||||||
PreparedMessages int `json:"prepared_messages"`
|
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||||
|
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||||
|
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||||
|
PreparedMessages int `json:"prepared_messages"`
|
||||||
|
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
|
|||||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
||||||
t.Fatalf("unexpected observation summary: %+v", got)
|
t.Fatalf("unexpected observation summary: %+v", got)
|
||||||
}
|
}
|
||||||
|
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
|
||||||
|
t.Fatalf("unexpected usage summary: %+v", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"go.maximumdirect.net/promptkit/internal/observation"
|
"go.maximumdirect.net/promptkit/internal/observation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ExecutionGroup collects facts from explicitly selected execution events.
|
// ExecutionGroup collects facts from selected execution events.
|
||||||
// Its zero value is ready for concurrent use. Do not copy it after first use.
|
// Its zero value is ready for concurrent use. Do not copy it after first use.
|
||||||
type ExecutionGroup struct{ group observation.Group }
|
type ExecutionGroup struct{ group observation.Group }
|
||||||
|
|
||||||
@@ -21,7 +21,8 @@ type ExecutionGroupMember struct {
|
|||||||
// member has a terminal, independently of usage completeness. An empty group
|
// member has a terminal, independently of usage completeness. An empty group
|
||||||
// has complete coverage and known complete zero usage; a group with members
|
// has complete coverage and known complete zero usage; a group with members
|
||||||
// but no terminals has unknown, incomplete usage. Overlapping groups are not
|
// but no terminals has unknown, incomplete usage. Overlapping groups are not
|
||||||
// generally additive.
|
// generally additive. CostUSD covers included terminals independently of
|
||||||
|
// terminal coverage, using the same known, complete, and source merge rules.
|
||||||
type ExecutionGroupSummary struct {
|
type ExecutionGroupSummary struct {
|
||||||
Members []ExecutionGroupMember `json:"members"`
|
Members []ExecutionGroupMember `json:"members"`
|
||||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||||
|
|||||||
@@ -112,6 +112,35 @@ func TestInjectedFailedResponseCostIsObservedOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCanceledDispatchCostFallbackAndUndispatchedEmptyCost(t *testing.T) {
|
||||||
|
configured := testCoordinator(t, func(*http.Request) (*http.Response, error) { t.Fatal("unexpected HTTP"); return nil, nil })
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
coordinator := NewInjected(configured.manager, cancelingCostClient{cancel: cancel})
|
||||||
|
var cost observation.CostAccumulator
|
||||||
|
req := testRequest(0)
|
||||||
|
req.Target.ZeroCost = true
|
||||||
|
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &cost), req); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("dispatch error=%v", err)
|
||||||
|
}
|
||||||
|
if got := cost.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}) {
|
||||||
|
t.Fatalf("dispatched=%+v", got)
|
||||||
|
}
|
||||||
|
var untouched observation.CostAccumulator
|
||||||
|
if _, err := coordinator.Generate(observation.WithCostAccumulator(ctx, &untouched), req); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("pre-dispatch error=%v", err)
|
||||||
|
}
|
||||||
|
if got := untouched.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||||
|
t.Fatalf("undispatched=%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type cancelingCostClient struct{ cancel context.CancelFunc }
|
||||||
|
|
||||||
|
func (c cancelingCostClient) Generate(ctx context.Context, _ domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
c.cancel()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
func TestObservedInterruptedWaitAndCanceledCapacity(t *testing.T) {
|
||||||
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
coordinator := testCoordinator(t, func(*http.Request) (*http.Response, error) { return response(503, `{"error":{}}`), nil })
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|||||||
@@ -18,8 +18,10 @@ import (
|
|||||||
type ExecutionObserver func(ExecutionEvent)
|
type ExecutionObserver func(ExecutionEvent)
|
||||||
|
|
||||||
// WithExecutionObserver adds an observer to executions using the returned
|
// WithExecutionObserver adds an observer to executions using the returned
|
||||||
// context. Inherited observers run before the newly added observer. A nil
|
// context. Request observers run first, then inherited contextual observers
|
||||||
// observer leaves the context unchanged.
|
// in registration order. A nil observer leaves the context unchanged.
|
||||||
|
// Prepared executions use their execution context, not their preparation
|
||||||
|
// context. Callers own context propagation and synchronization of observers.
|
||||||
func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context {
|
func WithExecutionObserver(ctx context.Context, observer ExecutionObserver) context.Context {
|
||||||
if observer == nil {
|
if observer == nil {
|
||||||
return ctx
|
return ctx
|
||||||
@@ -102,7 +104,9 @@ type ObservedTokenUsage struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CacheHitRatio returns the fraction of input tokens served from cache when
|
// CacheHitRatio returns the fraction of input tokens served from cache when
|
||||||
// both counts are known, complete, and consistent. Cache writes are independent.
|
// both counts are known, complete, and consistent, with positive input and
|
||||||
|
// reads between zero and input inclusive. Otherwise it returns (0, false).
|
||||||
|
// Cache writes are independent; aggregate ratios use summed counts.
|
||||||
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
|
func (u ObservedTokenUsage) CacheHitRatio() (float64, bool) {
|
||||||
input, reads := u.PromptTokens, u.CachedTokens
|
input, reads := u.PromptTokens, u.CachedTokens
|
||||||
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
|
if !input.Known || !input.Complete || !reads.Known || !reads.Complete || input.Value <= 0 || reads.Value < 0 || reads.Value > input.Value {
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
|||||||
// A successful PrepareExecution may retain an Observer function in this
|
// A successful PrepareExecution may retain an Observer function in this
|
||||||
// handle's private payload. The callback and other private execution settings
|
// handle's private payload. The callback and other private execution settings
|
||||||
// are cleared on claim completion or Discard. Details never exposes them.
|
// are cleared on claim completion or Discard. Details never exposes them.
|
||||||
|
// Contextual observers are selected from the RunPrepared context, not retained
|
||||||
|
// from preparation.
|
||||||
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
|
// It also retains RunRequest.CaptureExchange and RunRequest.ContinueFrom for
|
||||||
// execution of the frozen snapshot. Only RunPrepared can produce its completed
|
// execution of the frozen snapshot. Only RunPrepared can produce its completed
|
||||||
// [Exchange]; preparation alone does not.
|
// [Exchange]; preparation alone does not.
|
||||||
|
|||||||
@@ -28,8 +28,10 @@ type ProfileInspection struct {
|
|||||||
// caller-owned. Nil pointers denote absent fields; non-nil pointers preserve
|
// caller-owned. Nil pointers denote absent fields; non-nil pointers preserve
|
||||||
// explicit zero values and empty collections in JSON and YAML.
|
// explicit zero values and empty collections in JSON and YAML.
|
||||||
type BackendInspection struct {
|
type BackendInspection struct {
|
||||||
BackendID string `json:"id" yaml:"id"`
|
BackendID string `json:"id" yaml:"id"`
|
||||||
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
|
// Provider is the normalized declaration, not a manufactured runtime default.
|
||||||
|
Provider *string `json:"provider,omitempty" yaml:"provider,omitempty"`
|
||||||
|
// ZeroCost preserves an explicitly declared false as well as true.
|
||||||
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
|
ZeroCost *bool `json:"zero_cost,omitempty" yaml:"zero_cost,omitempty"`
|
||||||
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
Endpoint string `json:"endpoint" yaml:"endpoint"`
|
||||||
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
APIKeyEnv *string `json:"api_key_env,omitempty" yaml:"api_key_env,omitempty"`
|
||||||
|
|||||||
6
types.go
6
types.go
@@ -574,7 +574,7 @@ type TokenUsage struct {
|
|||||||
ReasoningTokens int `json:"reasoning_tokens"`
|
ReasoningTokens int `json:"reasoning_tokens"`
|
||||||
// TotalTokens is the reported total-token count.
|
// TotalTokens is the reported total-token count.
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
// CachedTokens is the reported cached-input-token count.
|
// CachedTokens is the reported input-token cache-read count.
|
||||||
CachedTokens int `json:"cached_tokens"`
|
CachedTokens int `json:"cached_tokens"`
|
||||||
// CacheWriteTokens is the reported cache-write-token count.
|
// CacheWriteTokens is the reported cache-write-token count.
|
||||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
@@ -660,7 +660,9 @@ type StructuredOutputJSONSpec struct {
|
|||||||
// treated as known and zero counts as unknown for observation only; the legacy
|
// treated as known and zero counts as unknown for observation only; the legacy
|
||||||
// interface cannot distinguish omitted zero from measured zero. An explicit
|
// interface cannot distinguish omitted zero from measured zero. An explicit
|
||||||
// UsagePresence can report known zero counts. Result usage sums the supplied
|
// UsagePresence can report known zero counts. Result usage sums the supplied
|
||||||
// Usage values independently of presence.
|
// Usage values independently of presence. A non-nil CostUSD is copied and
|
||||||
|
// retained for attempt accounting even when returned alongside an error;
|
||||||
|
// non-finite values are treated as unknown before configured-zero fallback.
|
||||||
type LLMClient interface {
|
type LLMClient interface {
|
||||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user