Adopt PromptKit v0.23.0 conversations and execution accounting

This commit is contained in:
2026-11-15 01:25:55 +00:00
parent d28b560265
commit d98d419b36
25 changed files with 689 additions and 924 deletions

View File

@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"sync"
"time"
@@ -47,13 +46,20 @@ func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hoo
return &RoleClient{runtime: r, role: role, profileID: profileID, settings: settings, hooks: hooks}, nil
}
func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
ctx, cancel := context.WithTimeout(parent, c.settings.OperationTimeout)
defer cancel()
type pendingCall struct {
request pk.RunRequest
finish func(*pk.RunResult, error) (structuredoutput.Response, error)
}
func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) {
var response structuredoutput.Response
var err error
var finishHooks []func()
ctx := parent
var terminalMu sync.Mutex
var terminal pk.ExecutionEvent
terminalAvailable := false
defer func() {
finishHooks = append(finishHooks, func() {
terminalMu.Lock()
if terminalAvailable {
response.Execution = outcomes.ExecutionCounts{
@@ -69,7 +75,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
}
}
terminalMu.Unlock()
}()
})
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
@@ -81,7 +87,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
record.Started = time.Now().UTC()
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
onCall(record)
defer func() {
finishHooks = append(finishHooks, func() {
record.Finished = time.Now().UTC()
if err != nil {
var failure *structuredoutput.Failure
@@ -92,123 +98,120 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
}
}
onCall(record)
}()
})
}
finish := func() {
for i := len(finishHooks) - 1; i >= 0; i-- {
finishHooks[i]()
}
}
fail := func(failure error) (*pendingCall, error) {
err = failure
finish()
return nil, failure
}
if parent.Err() != nil {
return response, classifiedFailure(parent, ctx, parent.Err(), nil)
return fail(classifiedFailure(parent, ctx, parent.Err(), nil))
}
if err = validateOutputTarget(out); err != nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err})
}
contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts)
if contractErr != nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr})
}
vars := prompts.WithSharedVariables(req.Variables)
for name, value := range prompts.SharedVariables() {
vars[name] = value
}
baseID, baseVars := req.PromptID, vars
var suffix []pk.RenderedMessage
if req.Replay != nil {
if req.Replay.Exchange == nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"}
var exchange *pk.Exchange
if req.ContinueFrom != nil {
var ok bool
exchange, ok = req.ContinueFrom.State.(*pk.Exchange)
if !ok || exchange == nil {
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
}
instruction, renderErr := c.runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars})
if renderErr != nil {
return response, classifiedFailure(parent, ctx, renderErr, nil)
}
response.RepairInstruction = convertMessages(instruction.Messages)
exchange := req.Replay.Exchange
baseID, baseVars = exchange.PromptID, exchange.Variables
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: exchange.FinalContent})
for _, message := range req.Replay.Turns {
native := pk.RenderedMessage{Role: message.Role, Content: message.Content}
if cache := message.CacheControl; cache != nil {
native.CacheControl = &pk.CacheControl{Type: pk.CacheControlType(cache.Type), TTL: cache.TTL}
}
suffix = append(suffix, native)
}
suffix = append(suffix, instruction.Messages...)
}
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
PromptID: baseID, ProfileID: c.profileID, Vars: baseVars, AppendedMessages: suffix,
runRequest := pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
TransientRetries: &c.settings.TransientRetries,
Validation: contract,
Observer: func(event pk.ExecutionEvent) {
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
if event.Kind == pk.ExecutionFinished {
terminalMu.Lock()
terminal = event
terminalAvailable = true
terminalMu.Unlock()
}
if c.hooks.OnEvent != nil {
c.hooks.OnEvent(event)
}
if c.hooks.OnEventWithCall != nil {
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
}
},
})
if prepErr != nil {
return response, classifiedFailure(parent, ctx, prepErr, nil)
}),
}
defer handle.Discard()
details := handle.Details()
if req.Replay != nil {
prefix := req.Replay.Exchange.Messages
if len(details.Messages) < len(prefix) || !reflect.DeepEqual(convertMessages(details.Messages[:len(prefix)]), prefix) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "repair conversation prefix changed"}
var details pk.PreparedRun
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
details = prepared
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
c.credentialNames = make(map[string]struct{})
}
c.credentialNames[name] = struct{}{}
c.credentialMu.Unlock()
}
response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model
response.Prepared = preparedSnapshot(details)
record.Prepared = response.Prepared
}
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
c.credentialNames = make(map[string]struct{})
runRequest.OperationTimeout = c.settings.OperationTimeout
runRequest.OnPrepared = capturePrepared
return &pendingCall{request: runRequest, finish: func(result *pk.RunResult, runErr error) (completed structuredoutput.Response, finalErr error) {
defer func() { err = finalErr; finish(); completed = response }()
if result != nil {
response.FinalContent = result.RawOutput
record.FinalContent = result.RawOutput
response.Model = result.ModelName
response.PromptTokens = result.Usage.PromptTokens
response.CompletionTokens = result.Usage.CompletionTokens
response.TotalTokens = result.Usage.TotalTokens
}
c.credentialNames[name] = struct{}{}
c.credentialMu.Unlock()
}
response.Prompt = promptMetadata(details)
record.Prompt = response.Prompt
response.Model = details.EffectiveModelParams.Model
response.Prepared = preparedSnapshot(details)
record.Prepared = response.Prepared
result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
if result != nil {
response.FinalContent = result.RawOutput
record.FinalContent = result.RawOutput
response.Model = result.ModelName
response.PromptTokens = result.Usage.PromptTokens
response.CompletionTokens = result.Usage.CompletionTokens
response.TotalTokens = result.Usage.TotalTokens
}
if runErr != nil {
return response, classifiedFailure(parent, ctx, runErr, result)
}
if result == nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
}
if !result.Validation.IsValid {
return response, classifiedFailure(parent, ctx, nil, result)
}
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
var typeError *json.UnmarshalTypeError
if errors.As(decodeErr, &typeError) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
if runErr != nil {
return response, classifiedFailure(parent, ctx, runErr, result)
}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
if req.CaptureExchange && req.Replay == nil {
frozen := make(map[string]string, len(vars))
for key, value := range vars {
frozen[key] = value
if result == nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
}
response.Exchange = &structuredoutput.Exchange{PromptID: req.PromptID, Variables: frozen, Messages: convertMessages(details.Messages), FinalContent: result.RawOutput}
if !result.Validation.IsValid {
return response, classifiedFailure(parent, ctx, nil, result)
}
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
var typeError *json.UnmarshalTypeError
if errors.As(decodeErr, &typeError) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
if result.Exchange != nil {
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
}
response.Content = json.RawMessage(result.RawOutput)
response.Provider = "openai-compatible"
return response, nil
}}, nil
}
func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent)
if err != nil {
return structuredoutput.Response{}, err
}
response.Content = json.RawMessage(result.RawOutput)
response.Provider = "openai-compatible"
return response, nil
result, err := c.runtime.Engine.Run(ctx, call.request)
return call.finish(result, err)
}
// OutputContract maps an Audita domain schema to the native request contract.
@@ -231,6 +234,12 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
failure.Cause = parent.Err()
return failure
}
if errors.Is(cause, pk.ErrOperationTimeout) {
failure.Class = structuredoutput.FailureRecoverable
failure.Reason = outcomes.ReasonOperationTimeout
failure.Message = "model operation timed out"
return failure
}
if isSetupFailure(cause) {
failure.Reason = outcomes.ReasonSetupFailure
failure.Message = "structured model setup failed"
@@ -298,10 +307,7 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
}
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
messages := make([]structuredoutput.Message, len(details.Messages))
for i, message := range details.Messages {
messages[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content}
}
messages := convertMessages(details.Messages)
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
}

View File

@@ -482,3 +482,14 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
t.Fatalf("unknown validation failure: %#v", failure)
}
}
// Preparation errors can carry the native operation deadline as their cause.
// That deadline is recoverable even when the outer category is a setup phase.
func TestOperationTimeoutDuringPreparationIsRecoverable(t *testing.T) {
ctx := context.Background()
cause := errors.Join(pk.ErrPromptLoad, pk.ErrOperationTimeout, context.DeadlineExceeded)
failure := classifiedFailure(ctx, ctx, cause, nil)
if failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(failure, pk.ErrOperationTimeout) {
t.Fatalf("failure = %#v", failure)
}
}

View File

@@ -19,141 +19,9 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
func TestNativeConversationReplayWithRepairContract(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": `profiles:
- id: proposal-profile
endpoint: https://example.invalid/v1
model: proposal-model
- id: repair-profile
endpoint: https://example.invalid/v1
model: repair-model
prompts:
- id: proposal-fixture
messages:
- role: system
content: 'System {{.Subject}}'
- role: user
content: 'Proposal {{.Subject}}'
output:
format: json
validation_mode: json_schema
schema_path: correction_set.v1.json
- id: repair-fixture
messages:
- role: user
content: 'Repair {{.Finding}}'
output:
format: text
validation_mode: none
`,
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
var requests []struct {
Model string `json:"model"`
Messages []struct{ Role, Content string } `json:"messages"`
ResponseFormat map[string]any `json:"response_format"`
}
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var request struct {
Model string `json:"model"`
Messages []struct{ Role, Content string } `json:"messages"`
ResponseFormat map[string]any `json:"response_format"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Fatal(err)
}
requests = append(requests, request)
content, _ := json.Marshal(outputs[len(requests)-1])
body := fmt.Sprintf(`{"model":%q,"choices":[{"message":{"content":%s}}]}`, request.Model, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
base := pk.RunRequest{PromptID: "proposal-fixture", ProfileID: "proposal-profile", Vars: map[string]string{"Subject": "source"}}
original, err := runtime.Engine.PrepareExecution(ctx, base)
if err != nil {
t.Fatal(err)
}
prefix := original.Details().Messages
initial, err := runtime.Engine.RunPrepared(ctx, original)
if err != nil {
t.Fatal(err)
}
suffix := []pk.RenderedMessage{{Role: pk.RoleAssistant, Content: initial.RawOutput}}
contract, err := OutputContract(responseschema.CorrectionRepairKey, 0)
if err != nil {
t.Fatal(err)
}
for _, finding := range []string{"first", "second"} {
instruction, err := runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: "repair-fixture", ProfileID: "repair-profile", Vars: map[string]string{"Finding": finding}})
if err != nil {
t.Fatal(err)
}
suffix = append(suffix, instruction.Messages...)
req := base
req.ProfileID = "repair-profile"
req.AppendedMessages = suffix
req.Validation = contract
handle, err := runtime.Engine.PrepareExecution(ctx, req)
if err != nil {
t.Fatal(err)
}
details := handle.Details()
if !reflect.DeepEqual(details.Messages[:len(prefix)], prefix) || !reflect.DeepEqual(details.Messages[len(prefix):], suffix) {
t.Fatalf("prepared conversation = %#v", details.Messages)
}
result, err := runtime.Engine.RunPrepared(ctx, handle)
if err != nil {
t.Fatal(err)
}
if !result.Validation.IsValid {
t.Fatalf("invalid result: %+v", result.Validation)
}
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: result.RawOutput})
}
if len(requests) != 3 || requests[0].Model != "proposal-model" || requests[1].Model != "repair-model" || requests[2].Model != "repair-model" {
t.Fatalf("models: %+v", requests)
}
for i, wantLen := range []int{2, 4, 6} {
req := requests[i]
if len(req.Messages) != wantLen {
t.Fatalf("request %d messages: %+v", i, req.Messages)
}
for j, message := range req.Messages {
if message.Role != string(append(prefix, suffix...)[j].Role) || message.Content != append(prefix, suffix...)[j].Content {
t.Fatalf("request %d message %d: %+v", i, j, message)
}
}
encoded, _ := json.Marshal(req.ResponseFormat)
expected := "corrections"
if i > 0 {
expected = "proposal_ref"
}
if !strings.Contains(string(encoded), expected) {
t.Fatalf("request %d contract: %s", i, encoded)
}
}
}
func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
@@ -207,17 +75,13 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
t.Fatalf("capture: %+v %v", initial, err)
}
vars["SectionJSON"] = "changed"
replay := &structuredoutput.Replay{Exchange: initial.Exchange}
exchange := initial.Exchange
for _, finding := range []string{"first", "second"} {
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, Replay: replay}, &map[string]any{})
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, ContinueFrom: exchange}, &map[string]any{})
if err != nil {
t.Fatal(err)
}
if len(result.RepairInstruction) != 1 {
t.Fatalf("instruction: %+v", result)
}
replay.Turns = append(replay.Turns, result.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: result.FinalContent})
exchange = result.Exchange
}
if !reflect.DeepEqual(models, []string{"proposal", "repair", "repair"}) {
t.Fatalf("models = %v", models)
@@ -235,21 +99,18 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
!strings.Contains(string(first.Content), `"text":"first"`) || !reflect.DeepEqual(first, messages[2][len(prefix)+1]) {
t.Fatalf("cached instruction changed during replay: first=%s replay=%s", first.Content, messages[2][len(prefix)+1].Content)
}
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, Replay: &structuredoutput.Replay{Exchange: initial.Exchange}}, &map[string]any{})
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, ContinueFrom: initial.Exchange}, &map[string]any{})
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
t.Fatalf("independent branch: %v %+v", err, messages)
}
bad := *initial.Exchange
bad.Messages = append([]structuredoutput.Message(nil), bad.Messages...)
bad.Messages[0].Content = "changed"
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, Replay: &structuredoutput.Replay{Exchange: &bad}}, &map[string]any{})
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, ContinueFrom: &structuredoutput.Exchange{}}, &map[string]any{})
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || len(messages) != 4 {
t.Fatalf("prefix failure: %v, calls %d", err, len(messages))
}
}
func TestRepairReplayUsesReplacementAndCountsStructuralRepair(t *testing.T) {
func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\n",
@@ -291,11 +152,11 @@ func TestRepairReplayUsesReplacementAndCountsStructuralRepair(t *testing.T) {
if err != nil {
t.Fatal(err)
}
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, Replay: &structuredoutput.Replay{Exchange: base.Exchange}}, &map[string]any{})
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, ContinueFrom: base.Exchange}, &map[string]any{})
if err != nil {
t.Fatal(err)
}
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] || len(result.RepairInstruction) != 1 || result.RepairInstruction[0].Content != "Replace fixture" {
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
t.Fatalf("result=%+v calls=%d", result, len(requests))
}
}

View File

@@ -0,0 +1,63 @@
package llm
import (
"context"
"encoding/json"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
var _ structuredoutput.Coordinator = (*RoleClient)(nil)
// Coordinate shares request construction and finalization with ordinary calls.
// PromptKit alone advances history and consumes the additional-turn budget.
func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
var exchange *pk.Exchange
if start != nil {
exchange, _ = start.State.(*pk.Exchange)
}
var pending *pendingCall
result, err := c.runtime.Engine.Coordinate(ctx, pk.CoordinateRequest{
Start: exchange, AdditionalTurns: budget, Observer: c.hooks.OnEvent,
Next: func(ctx context.Context, turn pk.ConversationTurn) (*pk.RunRequest, error) {
var delivered structuredoutput.Turn
if turn.Result != nil {
response, failure := pending.finish(turn.Result, nil)
pending = nil
delivered = structuredoutput.Turn{Response: &response, Err: failure}
}
selected, err := next(ctx, delivered)
if err != nil || selected == nil {
return nil, err
}
// Exhaustion needs a requested turn, but must not allocate a logical call.
if turn.RemainingTurns == 0 {
return &pk.RunRequest{}, nil
}
var output json.RawMessage
pending, err = c.prepareCall(ctx, *selected, &output, nil)
if err != nil {
_, domainErr := next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: err})
if domainErr != nil {
return nil, domainErr
}
return nil, err
}
return &pending.request, nil
},
})
if pending != nil {
// Execution errors and withheld responses do not enter Next. Finalize their
// diagnostic evidence and let the domain record the terminal failure once.
response, failure := pending.finish(result.Undelivered, err)
_, domainErr := next(ctx, structuredoutput.Turn{Response: &response, Err: failure})
if domainErr != nil {
return false, domainErr
}
if structuredoutput.IsRecoverable(failure) {
return false, nil
}
}
return result.Outcome == pk.CoordinationExhausted, err
}

View File

@@ -0,0 +1,141 @@
package llm
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "go.maximumdirect.net/promptkit"
)
func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
for _, tc := range []struct {
name string
reply reply
again bool
reason outcomes.ReasonCode
}{
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}},
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, again: true},
{name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted},
{name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed},
} {
t.Run(tc.name, func(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, tc.reply}, 0, 0, nil)
collector := NewObservationCollector()
client, err := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
if err != nil {
t.Fatal(err)
}
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
deliveries := 0
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
deliveries++
if tc.reason != "" {
var failure *structuredoutput.Failure
if !errors.As(turn.Err, &failure) || failure.Reason != tc.reason {
t.Fatalf("failure=%v", turn.Err)
}
return nil, nil
}
if turn.Err != nil || len(turn.Response.Content) == 0 {
t.Fatalf("turn=%+v", turn)
}
if !tc.again {
return nil, nil
}
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
}
snapshot := collector.Snapshot()
summary := Summarize(snapshot)
if len(snapshot.Calls) != 2 || summary.ProviderAttempts != 2 || !summary.TerminalCoverageComplete {
t.Fatalf("summary=%+v calls=%+v", summary, snapshot.Calls)
}
for _, call := range snapshot.Calls {
if call.Finished.IsZero() {
t.Fatal("unfinished logical call")
}
}
})
}
}
func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
hooks := collector.Hooks()
hooks.OnEvent = func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionFinished {
cancel()
}
}
client, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, hooks)
deliveries := 0
_, err = client.Coordinate(ctx, base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
deliveries++
if !errors.Is(turn.Err, context.Canceled) || turn.Response.Prepared == nil || turn.Response.FinalContent == "" || len(turn.Response.Content) != 0 {
t.Fatalf("canceled turn=%+v", turn)
}
return nil, turn.Err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
if !errors.Is(err, context.Canceled) || deliveries != 1 {
t.Fatalf("deliveries=%d err=%v", deliveries, err)
}
if got := Summarize(collector.Snapshot()); got.ProviderAttempts != 2 || !got.TerminalCoverageComplete {
t.Fatalf("summary=%+v", got)
}
}
func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
validation, _ := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, collector.Hooks())
req := adapterRequest(responseschema.CorrectionSetKey)
req.CaptureExchange = true
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
if err != nil {
t.Fatal(err)
}
_, err = repair.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response != nil {
if turn.Err != nil {
return nil, turn.Err
}
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
return nil, err
}
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
})
summary := Summarize(collector.Snapshot())
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
t.Fatalf("calls=%d summary=%+v err=%v", count.Load(), summary, err)
}
}

View File

@@ -115,7 +115,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
capturing := request
capturing.CaptureExchange = true
response, err := client.CompleteStructured(context.Background(), capturing, &result)
if tc.success && (response.Exchange == nil || response.Exchange.FinalContent != tc.outputs[len(tc.outputs)-1]) {
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
t.Fatalf("captured final response: %+v", response.Exchange)
}
if (err == nil) != tc.success || calls != tc.wantCalls {

View File

@@ -64,21 +64,24 @@ type LogicalCall struct {
}
type ObservationSnapshot struct {
Calls []LogicalCall `json:"calls"`
Events []ExecutionObservation `json:"events"`
Group *pk.ExecutionGroupSummary `json:"-"`
Calls []LogicalCall `json:"calls"`
Events []ExecutionObservation `json:"events"`
}
type ExecutionSummary struct {
LogicalCalls uint64 `json:"logical_calls"`
ProviderAttempts uint64 `json:"provider_attempts"`
TransientRetries uint64 `json:"transient_retries"`
StructuralRepairs uint64 `json:"structural_repairs"`
Usage Usage `json:"usage"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
LogicalCalls uint64 `json:"logical_calls"`
ProviderAttempts uint64 `json:"provider_attempts"`
TransientRetries uint64 `json:"transient_retries"`
StructuralRepairs uint64 `json:"structural_repairs"`
Usage Usage `json:"usage"`
}
// ObservationCollector accepts short synchronous callbacks from parallel calls.
// Snapshot should be taken after the runner has joined active section work.
type ObservationCollector struct {
group pk.ExecutionGroup
mu sync.Mutex
calls map[uint64]LogicalCall
events []ExecutionObservation
@@ -137,6 +140,7 @@ func (c *ObservationCollector) recordEvent(callID uint64, context structuredoutp
if c == nil {
return
}
c.group.Observe(event)
count := func(value pk.ObservedTokenCount) TokenCount {
return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
}
@@ -176,52 +180,28 @@ func (c *ObservationCollector) Snapshot() ObservationSnapshot {
}
return snapshot.Events[i].Time.Before(snapshot.Events[j].Time)
})
group := c.group.Snapshot()
snapshot.Group = &group
return snapshot
}
// Summarize counts terminal usage once per logical call. PromptKit's terminal
// usage already aggregates its attempts, retries, and repairs.
// Summarize delegates deduplication and terminal accounting to PromptKit. A
// filtered diagnostic view selects its own group; overlapping views are not additive.
func Summarize(snapshot ObservationSnapshot) ExecutionSummary {
result := ExecutionSummary{LogicalCalls: uint64(len(snapshot.Calls))}
terminal := make(map[uint64]ExecutionObservation)
for _, event := range snapshot.Events {
if event.Kind == string(pk.ExecutionFinished) {
terminal[event.CallID] = event
summary := snapshot.Group
if summary == nil {
var group pk.ExecutionGroup
count := func(c TokenCount) pk.ObservedTokenCount {
return pk.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
}
}
dispatched := false
aggregate := func(dst *TokenCount, source TokenCount) {
if source.Known {
dst.Value += source.Value
dst.Known = true
for _, event := range snapshot.Events {
group.Observe(pk.ExecutionEvent{Kind: pk.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.StructuralRepairs, Usage: pk.ObservedTokenUsage{PromptTokens: count(event.Usage.Prompt), CompletionTokens: count(event.Usage.Completion), TotalTokens: count(event.Usage.Total), CachedTokens: count(event.Usage.Cached), CacheWriteTokens: count(event.Usage.CacheWrite)}})
}
dst.Complete = dst.Complete && source.Complete
value := group.Snapshot()
summary = &value
}
result.Usage = Usage{Prompt: TokenCount{Complete: true}, Completion: TokenCount{Complete: true}, Total: TokenCount{Complete: true}, Cached: TokenCount{Complete: true}, CacheWrite: TokenCount{Complete: true}}
for _, call := range snapshot.Calls {
event, ok := terminal[call.CallID]
if !ok {
continue
}
result.ProviderAttempts += event.Attempts
result.TransientRetries += event.TransientRetries
result.StructuralRepairs += event.StructuralRepairs
if event.Attempts == 0 {
continue
}
dispatched = true
aggregate(&result.Usage.Prompt, event.Usage.Prompt)
aggregate(&result.Usage.Completion, event.Usage.Completion)
aggregate(&result.Usage.Total, event.Usage.Total)
aggregate(&result.Usage.Cached, event.Usage.Cached)
aggregate(&result.Usage.CacheWrite, event.Usage.CacheWrite)
count := func(c pk.ObservedTokenCount) TokenCount {
return TokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
}
if !dispatched {
result.Usage.Prompt.Known = true
result.Usage.Completion.Known = true
result.Usage.Total.Known = true
result.Usage.Cached.Known = true
result.Usage.CacheWrite.Known = true
}
return result
return ExecutionSummary{LogicalCalls: uint64(len(snapshot.Calls)), TerminalCoverageComplete: summary.TerminalCoverageComplete, ProviderAttempts: summary.Attempts, TransientRetries: summary.TransientRetries, StructuralRepairs: summary.RepairCalls, Usage: Usage{Prompt: count(summary.Usage.PromptTokens), Completion: count(summary.Usage.CompletionTokens), Total: count(summary.Usage.TotalTokens), Cached: count(summary.Usage.CachedTokens), CacheWrite: count(summary.Usage.CacheWriteTokens)}}
}

View File

@@ -3,6 +3,7 @@ package llm
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
@@ -23,8 +24,8 @@ func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
}
call(1) // preparation failed before PromptKit emitted events
call(2) // capacity rejected without a dispatched attempt
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{Kind: pk.ExecutionAttempt, Dispatched: false})
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 0})
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{RunID: "run-2", Kind: pk.ExecutionAttempt, Dispatched: false})
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{RunID: "run-2", Kind: pk.ExecutionFinished, Attempts: 0, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CompletionTokens: pk.ObservedTokenCount{Known: true, Complete: true}, TotalTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CachedTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CacheWriteTokens: pk.ObservedTokenCount{Known: true, Complete: true}}})
summary := Summarize(collector.Snapshot())
if summary.LogicalCalls != 2 || summary.ProviderAttempts != 0 || summary.Usage.Total != (TokenCount{Known: true, Complete: true}) {
t.Fatalf("zero-attempt accounting: %+v", summary)
@@ -32,16 +33,16 @@ func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
call(3)
context := structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar", ValidatorName: "editorial_review"}
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{Kind: pk.ExecutionAttempt, Dispatched: true, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 100, Known: true, Complete: true}}})
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 3, TransientRetries: 1, RepairCalls: 1, Usage: pk.ObservedTokenUsage{
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{RunID: "run-3", Kind: pk.ExecutionAttempt, Dispatched: true, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 100, Known: true, Complete: true}}})
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{RunID: "run-3", Kind: pk.ExecutionFinished, Attempts: 3, TransientRetries: 1, RepairCalls: 1, Usage: pk.ObservedTokenUsage{
PromptTokens: pk.ObservedTokenCount{Value: 5, Known: true, Complete: false},
TotalTokens: pk.ObservedTokenCount{Value: 8, Known: true, Complete: false},
}})
call(4)
hooks.OnEventWithCall(4, context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: false, Complete: false}, TotalTokens: pk.ObservedTokenCount{Known: false, Complete: false}}})
hooks.OnEventWithCall(4, context, pk.ExecutionEvent{RunID: "run-4", Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: false, Complete: false}, TotalTokens: pk.ObservedTokenCount{Known: false, Complete: false}}})
snapshot := collector.Snapshot()
summary = Summarize(snapshot)
if summary.LogicalCalls != 4 || summary.ProviderAttempts != 4 || summary.TransientRetries != 1 || summary.StructuralRepairs != 1 || summary.Usage.Total.Value != 8 || !summary.Usage.Total.Known || summary.Usage.Total.Complete || summary.Usage.Prompt.Value != 5 || !summary.Usage.Prompt.Known || summary.Usage.Prompt.Complete || summary.Usage.Cached.Known || summary.Usage.Cached.Complete {
if summary.LogicalCalls != 4 || summary.ProviderAttempts != 4 || summary.TransientRetries != 1 || summary.StructuralRepairs != 1 || summary.Usage.Total.Value != 8 || !summary.Usage.Total.Known || summary.Usage.Total.Complete || summary.Usage.Prompt.Value != 5 || !summary.Usage.Prompt.Known || summary.Usage.Prompt.Complete || !summary.Usage.Cached.Known || summary.Usage.Cached.Complete {
t.Fatalf("partial usage or double count: %+v", summary)
}
raw, err := json.Marshal(snapshot)
@@ -68,7 +69,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T)
context := structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar", SectionIndex: &index}
started := time.Now().UTC()
hooks.OnCall(structuredoutput.CallRecord{CallID: uint64(id), Context: context, Started: started})
hooks.OnEventWithCall(uint64(id), context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, RunID: "run", Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 1, Known: true, Complete: true}}})
hooks.OnEventWithCall(uint64(id), context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, RunID: fmt.Sprint(id), Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 1, Known: true, Complete: true}}})
hooks.OnCall(structuredoutput.CallRecord{CallID: uint64(id), Context: context, Started: started, Finished: time.Now().UTC()})
index = -1
}(i)
@@ -124,3 +125,18 @@ func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) {
t.Fatalf("logical failures: %+v", snapshot.Calls)
}
}
func TestObservationCoverageIsIndependentOfUsage(t *testing.T) {
collector := NewObservationCollector()
hooks := collector.Hooks()
hooks.OnEventWithCall(1, structuredoutput.DiagnosticContext{}, pk.ExecutionEvent{RunID: "pending", Kind: pk.ExecutionAttempt})
got := Summarize(collector.Snapshot())
if got.TerminalCoverageComplete || got.Usage.Total.Known || got.Usage.Total.Complete {
t.Fatalf("incomplete group=%+v", got)
}
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{}, pk.ExecutionEvent{RunID: "finished", Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 7, Known: true, Complete: true}}})
got = Summarize(collector.Snapshot())
if got.TerminalCoverageComplete || !got.Usage.Total.Known || !got.Usage.Total.Complete || got.Usage.Total.Value != 7 {
t.Fatalf("partial coverage=%+v", got)
}
}

View File

@@ -20,15 +20,16 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
}
type executionCall struct {
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`
Prompt structuredoutput.PromptMetadata `json:"prompt"`
Started time.Time `json:"started"`
Finished *time.Time `json:"finished,omitempty"`
ElapsedMS int64 `json:"elapsed_ms"`
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
Summary reporting.LLMSummary `json:"summary"`
}
type executionEvent struct {
@@ -72,7 +73,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
value := call.Finished
finished = &value
}
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
}
for _, event := range snapshot.Events {
var generation, attempt *uint64
@@ -87,7 +88,8 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
}
return struct {
Calls []executionCall `json:"calls"`
Events []executionEvent `json:"events"`
}{calls, events}
Summary llm.ExecutionSummary `json:"summary"`
Calls []executionCall `json:"calls"`
Events []executionEvent `json:"events"`
}{llm.Summarize(snapshot), calls, events}
}

View File

@@ -19,7 +19,7 @@ func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
},
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
Events: []llm.ExecutionObservation{{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
}
raw, err := json.Marshal(BuildLLMExecution(snapshot))
if err != nil {
@@ -46,7 +46,7 @@ func BenchmarkExecutionReports(b *testing.B) {
snapshot := llm.ObservationSnapshot{}
for i := 0; i < n; i++ {
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{RunID: fmt.Sprint(i), CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
}
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
@@ -64,9 +64,9 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
},
Events: []llm.ExecutionObservation{
{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
{CallID: 3, RunID: "run-3", Kind: string(pk.ExecutionFinished), Attempts: 1},
{CallID: 2, RunID: "run-2", Kind: string(pk.ExecutionFinished), Attempts: 1},
},
}
raw, err := json.Marshal(BuildLLMExecution(snapshot))

View File

@@ -18,7 +18,7 @@ import (
)
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
exchange := &structuredoutput.Exchange{PromptID: "captured", FinalContent: "response"}
exchange := &structuredoutput.Exchange{}
client := &fakeStructuredClient{
completion: structuredoutput.Response{Exchange: exchange},
responses: []StructuredCorrectionSet{

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
"encoding/json"
"errors"
"fmt"
"path/filepath"
@@ -117,8 +118,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
return findings
}
findings := safeFindings(rejected)
replay := &structuredoutput.Replay{Exchange: input.Exchange}
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
if !ok {
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
}
buildRequest := func() (*structuredoutput.Request, error) {
request := proposal_generation.RepairRequest{
ProposalRef: position + 1,
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
@@ -134,12 +138,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
payload, err := request.JSON()
if err != nil {
branches = append(branches, branch)
return finish(err)
return nil, err
}
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
}
exhausted, coordinationErr := coordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response == nil {
return buildRequest()
}
completion, callErr := *turn.Response, turn.Err
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
var response proposal_generation.RepairResponse
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
if callErr == nil {
_ = json.Unmarshal(completion.Content, &response)
}
operation := outcomes.Operation{
Reference: outcomes.Reference{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
@@ -183,26 +198,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
if structuredoutput.IsRecoverable(callErr) {
branch.Outcome = "unavailable"
break
return nil, nil
}
branches = append(branches, branch)
return finish(callErr)
return nil, callErr
}
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
if err != nil {
operation.FailureReason = outcomes.ReasonInvalidOutput
finishAttempt(err, nil, validateSectionCandidatesResult{})
branch.Outcome = "unavailable"
break
return nil, nil
}
operation.Succeeded = true
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
if decoded.Action == "withdraw" {
finishAttempt(nil, nil, validateSectionCandidatesResult{})
branch.Outcome = "withdrawn"
break
return nil, nil
}
revision := decoded.Correction
localRef = revision.SegmentReference
@@ -225,7 +237,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
}
for i := range validated.operations {
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
proposalIndex, attemptNumber := original.ProposalIndex, attemptNumber
validated.operations[i].Reference.ProposalIndex = &proposalIndex
validated.operations[i].Reference.RepairAttempt = &attemptNumber
}
@@ -236,32 +248,35 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
finishAttempt(validationErr, &current, validated)
initial.operations = append(initial.operations, validated.operations...)
if validationErr != nil {
branches = append(branches, branch)
return finish(validationErr)
return nil, validationErr
}
if approved {
branch.Outcome = "approved"
initial.approved = append(initial.approved, current)
superseded[original.ProposalIndex] = true
break
return nil, nil
}
if actionable(original.ProposalIndex, validated.rejected) {
if attempt+1 == input.Spec.Settings.LLM.CorrectionRepairAttempts {
branch.Outcome = "exhausted"
break
}
findings = safeFindings(validated.rejected)
continue
return buildRequest()
}
branch.Outcome = "unavailable"
for _, d := range validated.rejected {
if d.Disposition != validators.DispositionUnavailable {
branch.Outcome = "rejected"
break
return nil, nil
}
}
break
return nil, nil
})
if exhausted {
branch.Outcome = "exhausted"
}
if coordinationErr != nil {
branches = append(branches, branch)
return finish(coordinationErr)
}
branches = append(branches, branch)
}
return finish(nil)

View File

@@ -4,10 +4,17 @@ import (
"context"
"encoding/json"
"errors"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
@@ -20,6 +27,7 @@ import (
)
type repairClient struct {
t *testing.T
outputs []string
calls []structuredoutput.Request
err error
@@ -27,11 +35,7 @@ type repairClient struct {
}
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
if req.Replay != nil {
copyReplay := *req.Replay
copyReplay.Turns = append([]structuredoutput.Message(nil), req.Replay.Turns...)
req.Replay = &copyReplay
}
c.calls = append(c.calls, req)
if c.err != nil {
return structuredoutput.Response{}, c.err
@@ -40,18 +44,18 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
if err := json.Unmarshal([]byte(value), out); err != nil {
return structuredoutput.Response{}, err
}
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
}
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -85,7 +89,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
@@ -93,7 +97,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
Exchange: &structuredoutput.Exchange{},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -125,7 +129,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
}
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
client := &repairClient{outputs: []string{
client := &repairClient{t: t, outputs: []string{
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
}}
@@ -138,7 +142,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
Exchange: &structuredoutput.Exchange{},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -175,9 +179,9 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{outputs: tc.outputs, err: tc.err}
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -189,8 +193,8 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
}
if tc.calls == 2 && len(client.calls[1].Replay.Turns) != 2 {
t.Fatalf("history: %+v", client.calls[1].Replay.Turns)
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
}
})
}
@@ -202,11 +206,11 @@ func (m repairModule) RepairPromptID() string { return "repair" }
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}, nil
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
@@ -246,11 +250,11 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}
return validators.Result{Decisions: decisions}, nil
}}
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -271,9 +275,9 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -285,12 +289,12 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
}
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -318,14 +322,14 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
warnings := 0
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -337,7 +341,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
}
func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
client := &repairClient{}
client := &repairClient{t: t}
for _, rejected := range [][]ValidatorRejectedChange{
{{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}},
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
@@ -352,10 +356,10 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
validation := &repairClient{secrets: []string{"validation-secret"}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
@@ -393,3 +397,66 @@ func rejectUntilRepair(preserved ...int) contracts.Validator {
return validators.Result{Decisions: decisions}, nil
}}
}
type repairTransport func(*http.Request) (*http.Response, error)
func (f repairTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
// Exercise the real adapter and upstream coordinator; only model responses are scripted.
func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
t := c.t
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
var selected *structuredoutput.Request
initial := true
runtime, err := llm.NewRuntime(ctx, loaded, &http.Client{Transport: repairTransport(func(*http.Request) (*http.Response, error) {
content := `{"corrections":[]}`
if initial {
initial = false
} else {
c.calls = append(c.calls, *selected)
content = c.outputs[len(c.calls)-1]
}
body, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}})
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body)))}, nil
})})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("repair", "fixture", config.LLM{OperationTimeout: time.Second}, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
base, err := client.CompleteStructured(ctx, structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, CaptureExchange: true, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}, &output)
if err != nil {
return false, err
}
return client.Coordinate(ctx, base.Exchange, budget, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
req, err := next(ctx, turn)
if req != nil && err == nil {
selected = req
if c.err != nil {
c.calls = append(c.calls, *req)
_, err = next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: c.err})
return nil, err
}
// Test prompt IDs vary; all use the same supplied-data fixture definition.
copy := *req
copy.PromptID = "repair"
return &copy, nil
}
return req, err
})
}

View File

@@ -31,7 +31,7 @@ type Request struct {
Schema responseschema.Key `json:"schema"`
Context DiagnosticContext `json:"context"`
CaptureExchange bool `json:"-"`
Replay *Replay `json:"-"`
ContinueFrom *Exchange `json:"-"`
}
// PromptMetadata is the execution snapshot returned by the runtime when available.
@@ -44,33 +44,23 @@ type PromptMetadata struct {
Model string `json:"model,omitempty"`
}
// Exchange holds a section's original prepared conversation for read-only replay.
// Exchange is an opaque immutable conversation owned by the runtime adapter.
type Exchange struct {
PromptID string
Variables map[string]string
Messages []Message
FinalContent string
}
// Replay carries a section exchange and the prior turns of one repair branch.
type Replay struct {
Exchange *Exchange
Turns []Message
State any
}
type Response struct {
Exchange *Exchange `json:"-"`
RepairInstruction []Message `json:"-"`
Execution outcomes.ExecutionCounts `json:"execution"`
Content json.RawMessage `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Prompt PromptMetadata `json:"prompt"`
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
FinalContent string `json:"final_content,omitempty"`
Exchange *Exchange `json:"-"`
Execution outcomes.ExecutionCounts `json:"execution"`
Content json.RawMessage `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Prompt PromptMetadata `json:"prompt"`
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
FinalContent string `json:"final_content,omitempty"`
}
// PreparedSnapshot is the exact rendered request snapshot. Its messages may
@@ -108,3 +98,16 @@ type CallRecord struct {
type Client interface {
CompleteStructured(ctx context.Context, req Request, out any) (Response, error)
}
// Turn delivers a completed correction request, or its failure. The initial
// callback has no response. A failure is terminal for this coordination.
type Turn struct {
Response *Response
Err error
}
// Coordinator delegates bounded conversation progression to the runtime.
// Returning no request stops the branch; requesting beyond the budget exhausts it.
type Coordinator interface {
Coordinate(context.Context, *Exchange, int, func(context.Context, Turn) (*Request, error)) (bool, error)
}