Document invocation observation and cost reporting
This commit is contained in:
@@ -49,7 +49,11 @@ func (t *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error)
|
||||
func modelResponse(content string, promptTokens, completionTokens int) *http.Response {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"choices": []any{map[string]any{"message": map[string]string{"content": content}}},
|
||||
"usage": map[string]int{"prompt_tokens": promptTokens, "completion_tokens": completionTokens},
|
||||
"usage": map[string]any{
|
||||
"prompt_tokens": promptTokens, "completion_tokens": completionTokens,
|
||||
"prompt_tokens_details": map[string]int{"cached_tokens": 2, "cache_write_tokens": 1},
|
||||
"completion_tokens_details": map[string]int{"reasoning_tokens": 1}, "cost": .01,
|
||||
},
|
||||
})
|
||||
return httpResponse(http.StatusOK, string(body))
|
||||
}
|
||||
@@ -80,6 +84,10 @@ type summary struct {
|
||||
ExhaustedOutcome promptkit.CoordinationOutcome `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts uint64 `json:"group_attempts"`
|
||||
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
@@ -97,6 +105,8 @@ func run() (summary, error) {
|
||||
filesystem := configkit.NewFilesystem("example schemas", assets)
|
||||
backend := promptkit.LocalBackend("https://example.invalid/v1", 1)
|
||||
backend.ID = backendID
|
||||
provider, zeroCost := "openrouter", true
|
||||
backend.Provider, backend.ZeroCost = &provider, &zeroCost
|
||||
inputs = append(inputs,
|
||||
configkit.Input{Source: promptkit.SchemaRootSource("example schemas", "schemas"), Destination: configkit.Keys("promptkit"), Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: ".", BaseDir: "."}},
|
||||
promptkit.SettingsInput(promptkit.BackendSource("example backend", []promptkit.BackendConfig{
|
||||
@@ -117,8 +127,9 @@ func run() (summary, error) {
|
||||
|
||||
var proposalLog eventLog
|
||||
var executions promptkit.ExecutionGroup
|
||||
proposal, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: promptkit.ComposeObservers(proposalLog.record, executions.Observe),
|
||||
ctx := promptkit.WithExecutionObserver(context.Background(), executions.Observe)
|
||||
proposal, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Reduce queue cleanup toil.")}, CaptureExchange: true, Observer: proposalLog.record,
|
||||
})
|
||||
if err != nil {
|
||||
return summary{}, err
|
||||
@@ -133,8 +144,8 @@ func run() (summary, error) {
|
||||
var assessmentLog eventLog
|
||||
var preparedDetails promptkit.PreparedRun
|
||||
decision := "rejected"
|
||||
coordination, err := engine.Coordinate(context.Background(), promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 1, Observer: executions.Observe,
|
||||
coordination, err := engine.Coordinate(ctx, promptkit.CoordinateRequest{
|
||||
Start: proposal.Exchange, AdditionalTurns: 1,
|
||||
Next: func(ctx context.Context, turn promptkit.ConversationTurn) (*promptkit.RunRequest, error) {
|
||||
if turn.Result == nil {
|
||||
return &promptkit.RunRequest{PromptID: "example.proposal", Inputs: map[string]promptkit.ArtifactRef{"idea": promptkit.Inline("Refine the proposal.")}, OperationTimeout: 10 * time.Second, OnPrepared: func(_ context.Context, details promptkit.PreparedRun) { preparedDetails = details }}, nil
|
||||
@@ -142,7 +153,7 @@ func run() (summary, error) {
|
||||
if !turn.Result.Validation.IsValid {
|
||||
return nil, fmt.Errorf("follow-up failed structural validation")
|
||||
}
|
||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: promptkit.ComposeObservers(assessmentLog.record, executions.Observe)})
|
||||
assessment, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "example.assessment", Inputs: map[string]promptkit.ArtifactRef{"proposal": promptkit.Inline(turn.Result.RawOutput)}, OperationTimeout: 10 * time.Second, Observer: assessmentLog.record})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -181,6 +192,8 @@ func run() (summary, error) {
|
||||
return summary{}, err
|
||||
}
|
||||
group := executions.Snapshot()
|
||||
_, ratioAvailable := group.Usage.CacheHitRatio()
|
||||
assessmentRatio, _ := assessmentTerminal.Usage.CacheHitRatio()
|
||||
return summary{
|
||||
ProposalAttempts: proposalTerminal.Attempts,
|
||||
ProposalTransientRetries: proposalTerminal.TransientRetries,
|
||||
@@ -192,6 +205,10 @@ func run() (summary, error) {
|
||||
ExhaustedOutcome: exhausted.Outcome,
|
||||
GroupMembers: len(group.Members),
|
||||
GroupAttempts: group.Attempts,
|
||||
GroupCost: group.CostUSD,
|
||||
GroupReasoning: group.Usage.ReasoningTokens,
|
||||
GroupCacheRatioAvailable: ratioAvailable,
|
||||
AssessmentCacheRatio: assessmentRatio,
|
||||
PreparedMessages: len(preparedDetails.Messages),
|
||||
PreparedParentMatches: preparedDetails.ParentRunID == proposal.RunID,
|
||||
}, nil
|
||||
|
||||
@@ -3,7 +3,10 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestObserveExample(t *testing.T) {
|
||||
@@ -12,18 +15,22 @@ func TestObserveExample(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome string `json:"coordination_outcome"`
|
||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts int `json:"group_attempts"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
ProposalAttempts *int `json:"proposal_attempts"`
|
||||
ProposalRetries *int `json:"proposal_transient_retries"`
|
||||
UsageComplete *bool `json:"proposal_usage_complete"`
|
||||
AssessmentAttempts *int `json:"assessment_attempts"`
|
||||
AssessmentRetries *int `json:"assessment_transient_retries"`
|
||||
Outcome string `json:"assessment_outcome"`
|
||||
CoordinationOutcome string `json:"coordination_outcome"`
|
||||
ExhaustedOutcome string `json:"exhausted_outcome"`
|
||||
GroupMembers int `json:"group_members"`
|
||||
GroupAttempts int `json:"group_attempts"`
|
||||
GroupCost promptkit.ObservedCost `json:"group_cost"`
|
||||
GroupReasoning promptkit.ObservedTokenCount `json:"group_reasoning"`
|
||||
GroupCacheRatioAvailable bool `json:"group_cache_ratio_available"`
|
||||
AssessmentCacheRatio float64 `json:"assessment_cache_ratio"`
|
||||
PreparedMessages int `json:"prepared_messages"`
|
||||
PreparedParentMatches bool `json:"prepared_parent_matches"`
|
||||
}
|
||||
if err := json.Unmarshal(output.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -33,4 +40,7 @@ func TestObserveExample(t *testing.T) {
|
||||
got.AssessmentRetries == nil || *got.AssessmentRetries != 0 || got.Outcome != "accepted" || got.CoordinationOutcome != "stopped" || got.ExhaustedOutcome != "exhausted" || got.GroupMembers != 3 || got.GroupAttempts != 4 || got.PreparedMessages < 3 || !got.PreparedParentMatches {
|
||||
t.Fatalf("unexpected observation summary: %+v", got)
|
||||
}
|
||||
if !got.GroupCost.Known || !got.GroupCost.Complete || got.GroupCost.Source != promptkit.CostSourceMixed || math.Abs(got.GroupCost.Value-.03) > 1e-12 || got.GroupReasoning.Value != 3 || got.GroupReasoning.Complete || got.GroupCacheRatioAvailable || math.Abs(got.AssessmentCacheRatio-2.0/7.0) > 1e-12 {
|
||||
t.Fatalf("unexpected usage summary: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user