Include conversation ancestry in execution observations

This commit is contained in:
2026-11-14 01:20:06 +00:00
parent 04569e44df
commit e5c42de06d
6 changed files with 79 additions and 2 deletions

View File

@@ -392,6 +392,8 @@ Do not add a separate preparation or validation algorithm for continuation.
## Stage 4 — Propagate Ancestry Through Ordinary Observations ## Stage 4 — Propagate Ancestry Through Ordinary Observations
**Status:** Complete.
**Scope:** immediate-parent metadata without accounting changes. **Scope:** immediate-parent metadata without accounting changes.
1. Add parent metadata to the internal recorder's event construction and the root 1. Add parent metadata to the internal recorder's event construction and the root

View File

@@ -514,6 +514,9 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
return e.run(ctx, req) return e.run(ctx, req)
} }
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) }) recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
}
recorder.Start() recorder.Start()
if idErr != nil { if idErr != nil {
err := fmt.Errorf("failed to create run id: %w", idErr) err := fmt.Errorf("failed to create run id: %w", idErr)

View File

@@ -16,6 +16,7 @@ import (
// calls methods in execution order; it has no shared history or background work. // calls methods in execution order; it has no shared history or background work.
type Recorder struct { type Recorder struct {
runID string runID string
parentRunID string
startedAt time.Time startedAt time.Time
now func() time.Time now func() time.Time
observer domain.ExecutionObserver observer domain.ExecutionObserver
@@ -95,6 +96,9 @@ func NewRunID() (string, error) {
// RunID returns this invocation's identity. // RunID returns this invocation's identity.
func (r *Recorder) RunID() string { return r.runID } func (r *Recorder) RunID() string { return r.runID }
// SetParentRunID records the immediate source identity before start delivery.
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
// Start delivers the opening event with only identities already resolved. // Start delivers the opening event with only identities already resolved.
func (r *Recorder) Start() { func (r *Recorder) Start() {
event := r.event(domain.ExecutionStarted) event := r.event(domain.ExecutionStarted)
@@ -177,7 +181,7 @@ func (r *Recorder) Finish(data FinishData) {
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent { func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
return domain.ExecutionEvent{ return domain.ExecutionEvent{
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID, Kind: kind, RunID: r.runID, ParentRunID: r.parentRunID, Time: r.now(), PromptID: r.selection.PromptID,
ProfileID: r.selection.ProfileID, ProfileID: r.selection.ProfileID,
BackendID: r.selection.BackendID, Model: r.selection.Model, BackendID: r.selection.BackendID, Model: r.selection.Model,
} }

View File

@@ -112,6 +112,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
if payload.observer != nil { if payload.observer != nil {
var idErr error var idErr error
recorder, idErr = observation.New(payload.observer) recorder, idErr = observation.New(payload.observer)
recorder.SetParentRunID(payload.parent.RunID())
observation.RegisterClaim(ctx, recorder) observation.RegisterClaim(ctx, recorder)
ctx = observation.WithRecorder(ctx, recorder) ctx = observation.WithRecorder(ctx, recorder)
selected := payload.prepared selected := payload.prepared

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"strings" "strings"
"testing" "testing"
@@ -15,6 +16,72 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client)) return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
} }
func TestContinuationObservationAncestry(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
engine := observationEngine(t, client)
ctx := context.Background()
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
var events []promptkit.ExecutionEvent
observer := func(event promptkit.ExecutionEvent) { events = append(events, event) }
req := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: observer}
prepared, err := engine.PrepareExecution(ctx, req)
if err != nil || len(events) != 0 {
t.Fatalf("preparation events=%#v error=%v", events, err)
}
check := func(result *promptkit.RunResult, runErr error) {
t.Helper()
if runErr != nil || result == nil || len(events) != 3 {
t.Fatalf("result=%#v error=%v events=%#v", result, runErr, events)
}
for _, event := range events {
if event.ParentRunID != initial.RunID || event.RunID != result.RunID || strings.Contains(fmt.Sprintf("%+v", event), "private answer") {
t.Fatalf("event = %#v", event)
}
}
events = nil
}
result, err := engine.Run(ctx, req)
check(result, err)
result, err = engine.RunPrepared(ctx, prepared)
check(result, err)
client.err = errors.New("failure")
if result, err := engine.Run(ctx, req); result != nil || err == nil || len(events) != 3 {
t.Fatalf("failure result=%#v error=%v events=%#v", result, err, events)
}
for _, event := range events {
if event.ParentRunID != initial.RunID {
t.Fatalf("failure event = %#v", event)
}
}
}
func TestContinuationNestedRunHasIndependentAncestry(t *testing.T) {
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
ctx := context.Background()
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
if err != nil {
t.Fatal(err)
}
var nested []promptkit.ExecutionEvent
var nestedErr error
_, err = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
_, nestedErr = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: func(value promptkit.ExecutionEvent) { nested = append(nested, value) }})
}
}})
if err != nil || nestedErr != nil || len(nested) == 0 {
t.Fatalf("errors=%v, %v; events=%#v", err, nestedErr, nested)
}
for _, event := range nested {
if event.ParentRunID != "" {
t.Fatalf("nested event = %#v", event)
}
}
}
func TestRunObservationCoversFacadeErrors(t *testing.T) { func TestRunObservationCoversFacadeErrors(t *testing.T) {
for _, tc := range []struct { for _, tc := range []struct {
name string name string

View File

@@ -130,7 +130,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete} return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
} }
return ExecutionEvent{ return ExecutionEvent{
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time, Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, ParentRunID: value.ParentRunID, Time: value.Time,
PromptID: value.PromptID, ProfileID: value.ProfileID, PromptID: value.PromptID, ProfileID: value.ProfileID,
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation, BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait, Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,