Define observed cost values and aggregation
This commit is contained in:
17
internal/domain/cost.go
Normal file
17
internal/domain/cost.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package domain
|
||||
|
||||
type CostSource string
|
||||
|
||||
const (
|
||||
CostSourceNone CostSource = ""
|
||||
CostSourceProvider CostSource = "provider"
|
||||
CostSourceConfiguredZero CostSource = "configured_zero"
|
||||
CostSourceMixed CostSource = "mixed"
|
||||
)
|
||||
|
||||
type ObservedCost struct {
|
||||
Value float64
|
||||
Known bool
|
||||
Complete bool
|
||||
Source CostSource
|
||||
}
|
||||
@@ -99,6 +99,7 @@ type RunResult struct {
|
||||
EffectiveModelParams ExecutionTarget
|
||||
InputHashes map[string]string
|
||||
Usage TokenUsage
|
||||
CostUSD ObservedCost
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Duration time.Duration
|
||||
@@ -328,6 +329,7 @@ type GenerateResponse struct {
|
||||
Content string
|
||||
Usage TokenUsage
|
||||
UsagePresence *TokenUsagePresence
|
||||
CostUSD *float64
|
||||
}
|
||||
|
||||
// TokenUsagePresence records which token fields a provider reported.
|
||||
|
||||
@@ -76,6 +76,7 @@ type ExecutionGroupSummary struct {
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
}
|
||||
|
||||
// ExecutionEvent contains metadata only; it carries no mutable containers or
|
||||
@@ -101,6 +102,7 @@ type ExecutionEvent struct {
|
||||
StatusCode int
|
||||
RetriesExhausted bool
|
||||
Usage ObservedTokenUsage
|
||||
CostUSD ObservedCost
|
||||
Attempts uint64
|
||||
TransientRetries uint64
|
||||
RepairCalls uint64
|
||||
|
||||
53
internal/observation/cost.go
Normal file
53
internal/observation/cost.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// ResolveCost applies reported-cost precedence over a configured-zero fallback.
|
||||
func ResolveCost(reported *float64, zeroCost bool) domain.ObservedCost {
|
||||
if reported != nil && !math.IsNaN(*reported) && !math.IsInf(*reported, 0) {
|
||||
return domain.ObservedCost{Value: *reported, Known: true, Complete: true, Source: domain.CostSourceProvider}
|
||||
}
|
||||
if zeroCost {
|
||||
return domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}
|
||||
}
|
||||
return domain.ObservedCost{}
|
||||
}
|
||||
|
||||
// CostAccumulator merges dispatched attempts or already aggregated costs.
|
||||
type CostAccumulator struct {
|
||||
value domain.ObservedCost
|
||||
started bool
|
||||
}
|
||||
|
||||
// Add incorporates one contribution.
|
||||
func (c *CostAccumulator) Add(next domain.ObservedCost) {
|
||||
if !c.started {
|
||||
c.started = true
|
||||
c.value = next
|
||||
return
|
||||
}
|
||||
if next.Known {
|
||||
c.value.Value += next.Value
|
||||
if next.Source != domain.CostSourceNone {
|
||||
if c.value.Source == domain.CostSourceNone {
|
||||
c.value.Source = next.Source
|
||||
} else if c.value.Source != next.Source {
|
||||
c.value.Source = domain.CostSourceMixed
|
||||
}
|
||||
}
|
||||
}
|
||||
c.value.Known = c.value.Known || next.Known
|
||||
c.value.Complete = c.value.Complete && next.Complete
|
||||
}
|
||||
|
||||
// Observed returns known complete zero when no contribution has been added.
|
||||
func (c CostAccumulator) Observed() domain.ObservedCost {
|
||||
if !c.started {
|
||||
return domain.ObservedCost{Known: true, Complete: true}
|
||||
}
|
||||
return c.value
|
||||
}
|
||||
63
internal/observation/cost_test.go
Normal file
63
internal/observation/cost_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package observation
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
func TestResolveCostPrecedence(t *testing.T) {
|
||||
zero, negative, invalid := 0.0, -1.5, math.Inf(1)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
reported *float64
|
||||
fallback bool
|
||||
want domain.ObservedCost
|
||||
}{
|
||||
{"unknown", nil, false, domain.ObservedCost{}},
|
||||
{"configured zero", nil, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"reported zero", &zero, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"negative", &negative, false, domain.ObservedCost{Value: -1.5, Known: true, Complete: true, Source: domain.CostSourceProvider}},
|
||||
{"invalid fallback", &invalid, true, domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceConfiguredZero}},
|
||||
{"invalid unknown", &invalid, false, domain.ObservedCost{}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := ResolveCost(tc.reported, tc.fallback); got != tc.want {
|
||||
t.Fatalf("got %+v want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCostAccumulatorMergesSourcesAndCoverage(t *testing.T) {
|
||||
var acc CostAccumulator
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{Known: true, Complete: true}) {
|
||||
t.Fatalf("empty: %+v", got)
|
||||
}
|
||||
acc.Add(domain.ObservedCost{})
|
||||
if got := acc.Observed(); got != (domain.ObservedCost{}) {
|
||||
t.Fatalf("unknown: %+v", got)
|
||||
}
|
||||
a, b := .1, .2
|
||||
acc.Add(ResolveCost(&a, false))
|
||||
acc.Add(ResolveCost(&b, false))
|
||||
acc.Add(ResolveCost(nil, true))
|
||||
got := acc.Observed()
|
||||
if math.Abs(got.Value-.3) > 1e-12 || !got.Known || got.Complete || got.Source != domain.CostSourceMixed {
|
||||
t.Fatalf("merged: %+v", got)
|
||||
}
|
||||
var zeros CostAccumulator
|
||||
zero := 0.0
|
||||
zeros.Add(ResolveCost(&zero, false))
|
||||
zeros.Add(ResolveCost(nil, true))
|
||||
if got := zeros.Observed(); got != (domain.ObservedCost{Known: true, Complete: true, Source: domain.CostSourceMixed}) {
|
||||
t.Fatalf("zeros: %+v", got)
|
||||
}
|
||||
var aggregate CostAccumulator
|
||||
aggregate.Add(domain.ObservedCost{Known: true, Complete: true})
|
||||
aggregate.Add(got)
|
||||
if aggregate.Observed() != got {
|
||||
t.Fatalf("aggregate: %+v", aggregate.Observed())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user