Include conversation ancestry in execution observations
This commit is contained in:
@@ -392,6 +392,8 @@ Do not add a separate preparation or validation algorithm for continuation.
|
|||||||
|
|
||||||
## Stage 4 — Propagate Ancestry Through Ordinary Observations
|
## Stage 4 — Propagate Ancestry Through Ordinary Observations
|
||||||
|
|
||||||
|
**Status:** Complete.
|
||||||
|
|
||||||
**Scope:** immediate-parent metadata without accounting changes.
|
**Scope:** immediate-parent metadata without accounting changes.
|
||||||
|
|
||||||
1. Add parent metadata to the internal recorder's event construction and the root
|
1. Add parent metadata to the internal recorder's event construction and the root
|
||||||
|
|||||||
@@ -514,6 +514,9 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
return e.run(ctx, req)
|
return e.run(ctx, req)
|
||||||
}
|
}
|
||||||
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
recorder, idErr := observation.New(func(event domain.ExecutionEvent) { req.Observer(fromDomainExecutionEvent(event)) })
|
||||||
|
if req.ContinueFrom != nil && req.ContinueFrom.internal != nil {
|
||||||
|
recorder.SetParentRunID(req.ContinueFrom.internal.RunID())
|
||||||
|
}
|
||||||
recorder.Start()
|
recorder.Start()
|
||||||
if idErr != nil {
|
if idErr != nil {
|
||||||
err := fmt.Errorf("failed to create run id: %w", idErr)
|
err := fmt.Errorf("failed to create run id: %w", idErr)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
// calls methods in execution order; it has no shared history or background work.
|
// calls methods in execution order; it has no shared history or background work.
|
||||||
type Recorder struct {
|
type Recorder struct {
|
||||||
runID string
|
runID string
|
||||||
|
parentRunID string
|
||||||
startedAt time.Time
|
startedAt time.Time
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
observer domain.ExecutionObserver
|
observer domain.ExecutionObserver
|
||||||
@@ -95,6 +96,9 @@ func NewRunID() (string, error) {
|
|||||||
// RunID returns this invocation's identity.
|
// RunID returns this invocation's identity.
|
||||||
func (r *Recorder) RunID() string { return r.runID }
|
func (r *Recorder) RunID() string { return r.runID }
|
||||||
|
|
||||||
|
// SetParentRunID records the immediate source identity before start delivery.
|
||||||
|
func (r *Recorder) SetParentRunID(id string) { r.parentRunID = id }
|
||||||
|
|
||||||
// Start delivers the opening event with only identities already resolved.
|
// Start delivers the opening event with only identities already resolved.
|
||||||
func (r *Recorder) Start() {
|
func (r *Recorder) Start() {
|
||||||
event := r.event(domain.ExecutionStarted)
|
event := r.event(domain.ExecutionStarted)
|
||||||
@@ -177,7 +181,7 @@ func (r *Recorder) Finish(data FinishData) {
|
|||||||
|
|
||||||
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
func (r *Recorder) event(kind domain.ExecutionEventKind) domain.ExecutionEvent {
|
||||||
return domain.ExecutionEvent{
|
return domain.ExecutionEvent{
|
||||||
Kind: kind, RunID: r.runID, Time: r.now(), PromptID: r.selection.PromptID,
|
Kind: kind, RunID: r.runID, ParentRunID: r.parentRunID, Time: r.now(), PromptID: r.selection.PromptID,
|
||||||
ProfileID: r.selection.ProfileID,
|
ProfileID: r.selection.ProfileID,
|
||||||
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
BackendID: r.selection.BackendID, Model: r.selection.Model,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,6 +112,7 @@ func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (
|
|||||||
if payload.observer != nil {
|
if payload.observer != nil {
|
||||||
var idErr error
|
var idErr error
|
||||||
recorder, idErr = observation.New(payload.observer)
|
recorder, idErr = observation.New(payload.observer)
|
||||||
|
recorder.SetParentRunID(payload.parent.RunID())
|
||||||
observation.RegisterClaim(ctx, recorder)
|
observation.RegisterClaim(ctx, recorder)
|
||||||
ctx = observation.WithRecorder(ctx, recorder)
|
ctx = observation.WithRecorder(ctx, recorder)
|
||||||
selected := payload.prepared
|
selected := payload.prepared
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -15,6 +16,72 @@ func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engi
|
|||||||
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestContinuationObservationAncestry(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "private answer"}}
|
||||||
|
engine := observationEngine(t, client)
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var events []promptkit.ExecutionEvent
|
||||||
|
observer := func(event promptkit.ExecutionEvent) { events = append(events, event) }
|
||||||
|
req := promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: observer}
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, req)
|
||||||
|
if err != nil || len(events) != 0 {
|
||||||
|
t.Fatalf("preparation events=%#v error=%v", events, err)
|
||||||
|
}
|
||||||
|
check := func(result *promptkit.RunResult, runErr error) {
|
||||||
|
t.Helper()
|
||||||
|
if runErr != nil || result == nil || len(events) != 3 {
|
||||||
|
t.Fatalf("result=%#v error=%v events=%#v", result, runErr, events)
|
||||||
|
}
|
||||||
|
for _, event := range events {
|
||||||
|
if event.ParentRunID != initial.RunID || event.RunID != result.RunID || strings.Contains(fmt.Sprintf("%+v", event), "private answer") {
|
||||||
|
t.Fatalf("event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
events = nil
|
||||||
|
}
|
||||||
|
result, err := engine.Run(ctx, req)
|
||||||
|
check(result, err)
|
||||||
|
result, err = engine.RunPrepared(ctx, prepared)
|
||||||
|
check(result, err)
|
||||||
|
client.err = errors.New("failure")
|
||||||
|
if result, err := engine.Run(ctx, req); result != nil || err == nil || len(events) != 3 {
|
||||||
|
t.Fatalf("failure result=%#v error=%v events=%#v", result, err, events)
|
||||||
|
}
|
||||||
|
for _, event := range events {
|
||||||
|
if event.ParentRunID != initial.RunID {
|
||||||
|
t.Fatalf("failure event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContinuationNestedRunHasIndependentAncestry(t *testing.T) {
|
||||||
|
engine := observationEngine(t, &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})
|
||||||
|
ctx := context.Background()
|
||||||
|
initial, err := engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", CaptureExchange: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var nested []promptkit.ExecutionEvent
|
||||||
|
var nestedErr error
|
||||||
|
_, err = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", ContinueFrom: initial.Exchange, Observer: func(event promptkit.ExecutionEvent) {
|
||||||
|
if event.Kind == promptkit.ExecutionFinished {
|
||||||
|
_, nestedErr = engine.Run(ctx, promptkit.RunRequest{PromptID: "prompt", Observer: func(value promptkit.ExecutionEvent) { nested = append(nested, value) }})
|
||||||
|
}
|
||||||
|
}})
|
||||||
|
if err != nil || nestedErr != nil || len(nested) == 0 {
|
||||||
|
t.Fatalf("errors=%v, %v; events=%#v", err, nestedErr, nested)
|
||||||
|
}
|
||||||
|
for _, event := range nested {
|
||||||
|
if event.ParentRunID != "" {
|
||||||
|
t.Fatalf("nested event = %#v", event)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunObservationCoversFacadeErrors(t *testing.T) {
|
func TestRunObservationCoversFacadeErrors(t *testing.T) {
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ func fromDomainExecutionEvent(value domain.ExecutionEvent) ExecutionEvent {
|
|||||||
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
return ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||||
}
|
}
|
||||||
return ExecutionEvent{
|
return ExecutionEvent{
|
||||||
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, Time: value.Time,
|
Kind: ExecutionEventKind(value.Kind), RunID: value.RunID, ParentRunID: value.ParentRunID, Time: value.Time,
|
||||||
PromptID: value.PromptID, ProfileID: value.ProfileID,
|
PromptID: value.PromptID, ProfileID: value.ProfileID,
|
||||||
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
BackendID: value.BackendID, Model: value.Model, Generation: value.Generation,
|
||||||
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
Attempt: value.Attempt, Dispatched: value.Dispatched, CapacityWait: value.CapacityWait,
|
||||||
|
|||||||
Reference in New Issue
Block a user