Adopt PromptKit v0.23.0 conversations and execution accounting
This commit is contained in:
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -47,13 +46,20 @@ func (r *Runtime) NewRoleClient(role, profileID string, settings config.LLM, hoo
|
||||
return &RoleClient{runtime: r, role: role, profileID: profileID, settings: settings, hooks: hooks}, nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) CompleteStructured(parent context.Context, req structuredoutput.Request, out any) (response structuredoutput.Response, err error) {
|
||||
ctx, cancel := context.WithTimeout(parent, c.settings.OperationTimeout)
|
||||
defer cancel()
|
||||
type pendingCall struct {
|
||||
request pk.RunRequest
|
||||
finish func(*pk.RunResult, error) (structuredoutput.Response, error)
|
||||
}
|
||||
|
||||
func (c *RoleClient) prepareCall(parent context.Context, req structuredoutput.Request, out any, observer pk.ExecutionObserver) (*pendingCall, error) {
|
||||
var response structuredoutput.Response
|
||||
var err error
|
||||
var finishHooks []func()
|
||||
ctx := parent
|
||||
var terminalMu sync.Mutex
|
||||
var terminal pk.ExecutionEvent
|
||||
terminalAvailable := false
|
||||
defer func() {
|
||||
finishHooks = append(finishHooks, func() {
|
||||
terminalMu.Lock()
|
||||
if terminalAvailable {
|
||||
response.Execution = outcomes.ExecutionCounts{
|
||||
@@ -69,7 +75,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
}
|
||||
}
|
||||
terminalMu.Unlock()
|
||||
}()
|
||||
})
|
||||
var record structuredoutput.CallRecord
|
||||
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
|
||||
record.CallID = c.runtime.nextCallID.Add(1)
|
||||
@@ -81,7 +87,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
record.Started = time.Now().UTC()
|
||||
record.Prompt = structuredoutput.PromptMetadata{PromptID: req.PromptID, ProfileID: c.profileID}
|
||||
onCall(record)
|
||||
defer func() {
|
||||
finishHooks = append(finishHooks, func() {
|
||||
record.Finished = time.Now().UTC()
|
||||
if err != nil {
|
||||
var failure *structuredoutput.Failure
|
||||
@@ -92,123 +98,120 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
|
||||
}
|
||||
}
|
||||
onCall(record)
|
||||
}()
|
||||
})
|
||||
}
|
||||
finish := func() {
|
||||
for i := len(finishHooks) - 1; i >= 0; i-- {
|
||||
finishHooks[i]()
|
||||
}
|
||||
}
|
||||
fail := func(failure error) (*pendingCall, error) {
|
||||
err = failure
|
||||
finish()
|
||||
return nil, failure
|
||||
}
|
||||
if parent.Err() != nil {
|
||||
return response, classifiedFailure(parent, ctx, parent.Err(), nil)
|
||||
return fail(classifiedFailure(parent, ctx, parent.Err(), nil))
|
||||
}
|
||||
if err = validateOutputTarget(out); err != nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err}
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "invalid structured output target", Cause: err})
|
||||
}
|
||||
contract, contractErr := OutputContract(req.Schema, c.settings.RepairAttempts)
|
||||
if contractErr != nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr})
|
||||
}
|
||||
vars := prompts.WithSharedVariables(req.Variables)
|
||||
for name, value := range prompts.SharedVariables() {
|
||||
vars[name] = value
|
||||
}
|
||||
baseID, baseVars := req.PromptID, vars
|
||||
var suffix []pk.RenderedMessage
|
||||
if req.Replay != nil {
|
||||
if req.Replay.Exchange == nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"}
|
||||
var exchange *pk.Exchange
|
||||
if req.ContinueFrom != nil {
|
||||
var ok bool
|
||||
exchange, ok = req.ContinueFrom.State.(*pk.Exchange)
|
||||
if !ok || exchange == nil {
|
||||
return fail(&structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"})
|
||||
}
|
||||
instruction, renderErr := c.runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars})
|
||||
if renderErr != nil {
|
||||
return response, classifiedFailure(parent, ctx, renderErr, nil)
|
||||
}
|
||||
response.RepairInstruction = convertMessages(instruction.Messages)
|
||||
exchange := req.Replay.Exchange
|
||||
baseID, baseVars = exchange.PromptID, exchange.Variables
|
||||
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: exchange.FinalContent})
|
||||
for _, message := range req.Replay.Turns {
|
||||
native := pk.RenderedMessage{Role: message.Role, Content: message.Content}
|
||||
if cache := message.CacheControl; cache != nil {
|
||||
native.CacheControl = &pk.CacheControl{Type: pk.CacheControlType(cache.Type), TTL: cache.TTL}
|
||||
}
|
||||
suffix = append(suffix, native)
|
||||
}
|
||||
suffix = append(suffix, instruction.Messages...)
|
||||
}
|
||||
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
|
||||
PromptID: baseID, ProfileID: c.profileID, Vars: baseVars, AppendedMessages: suffix,
|
||||
runRequest := pk.RunRequest{
|
||||
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
|
||||
CaptureExchange: req.CaptureExchange, ContinueFrom: exchange,
|
||||
TransientRetries: &c.settings.TransientRetries,
|
||||
Validation: contract,
|
||||
Observer: func(event pk.ExecutionEvent) {
|
||||
Observer: pk.ComposeObservers(observer, func(event pk.ExecutionEvent) {
|
||||
if event.Kind == pk.ExecutionFinished {
|
||||
terminalMu.Lock()
|
||||
terminal = event
|
||||
terminalAvailable = true
|
||||
terminalMu.Unlock()
|
||||
}
|
||||
if c.hooks.OnEvent != nil {
|
||||
c.hooks.OnEvent(event)
|
||||
}
|
||||
if c.hooks.OnEventWithCall != nil {
|
||||
c.hooks.OnEventWithCall(record.CallID, record.Context, event)
|
||||
}
|
||||
},
|
||||
})
|
||||
if prepErr != nil {
|
||||
return response, classifiedFailure(parent, ctx, prepErr, nil)
|
||||
}),
|
||||
}
|
||||
defer handle.Discard()
|
||||
details := handle.Details()
|
||||
if req.Replay != nil {
|
||||
prefix := req.Replay.Exchange.Messages
|
||||
if len(details.Messages) < len(prefix) || !reflect.DeepEqual(convertMessages(details.Messages[:len(prefix)]), prefix) {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "repair conversation prefix changed"}
|
||||
var details pk.PreparedRun
|
||||
capturePrepared := func(_ context.Context, prepared pk.PreparedRun) {
|
||||
details = prepared
|
||||
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
|
||||
c.credentialMu.Lock()
|
||||
if c.credentialNames == nil {
|
||||
c.credentialNames = make(map[string]struct{})
|
||||
}
|
||||
c.credentialNames[name] = struct{}{}
|
||||
c.credentialMu.Unlock()
|
||||
}
|
||||
response.Prompt = promptMetadata(details)
|
||||
record.Prompt = response.Prompt
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
response.Prepared = preparedSnapshot(details)
|
||||
record.Prepared = response.Prepared
|
||||
}
|
||||
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
|
||||
c.credentialMu.Lock()
|
||||
if c.credentialNames == nil {
|
||||
c.credentialNames = make(map[string]struct{})
|
||||
runRequest.OperationTimeout = c.settings.OperationTimeout
|
||||
runRequest.OnPrepared = capturePrepared
|
||||
return &pendingCall{request: runRequest, finish: func(result *pk.RunResult, runErr error) (completed structuredoutput.Response, finalErr error) {
|
||||
defer func() { err = finalErr; finish(); completed = response }()
|
||||
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
record.FinalContent = result.RawOutput
|
||||
response.Model = result.ModelName
|
||||
response.PromptTokens = result.Usage.PromptTokens
|
||||
response.CompletionTokens = result.Usage.CompletionTokens
|
||||
response.TotalTokens = result.Usage.TotalTokens
|
||||
}
|
||||
c.credentialNames[name] = struct{}{}
|
||||
c.credentialMu.Unlock()
|
||||
}
|
||||
response.Prompt = promptMetadata(details)
|
||||
record.Prompt = response.Prompt
|
||||
response.Model = details.EffectiveModelParams.Model
|
||||
response.Prepared = preparedSnapshot(details)
|
||||
record.Prepared = response.Prepared
|
||||
result, runErr := c.runtime.Engine.RunPrepared(ctx, handle)
|
||||
if result != nil {
|
||||
response.FinalContent = result.RawOutput
|
||||
record.FinalContent = result.RawOutput
|
||||
response.Model = result.ModelName
|
||||
response.PromptTokens = result.Usage.PromptTokens
|
||||
response.CompletionTokens = result.Usage.CompletionTokens
|
||||
response.TotalTokens = result.Usage.TotalTokens
|
||||
}
|
||||
if runErr != nil {
|
||||
return response, classifiedFailure(parent, ctx, runErr, result)
|
||||
}
|
||||
if result == nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
return response, classifiedFailure(parent, ctx, nil, result)
|
||||
}
|
||||
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
||||
var typeError *json.UnmarshalTypeError
|
||||
if errors.As(decodeErr, &typeError) {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
|
||||
if runErr != nil {
|
||||
return response, classifiedFailure(parent, ctx, runErr, result)
|
||||
}
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
||||
}
|
||||
if req.CaptureExchange && req.Replay == nil {
|
||||
frozen := make(map[string]string, len(vars))
|
||||
for key, value := range vars {
|
||||
frozen[key] = value
|
||||
if result == nil {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "structured model call returned no result"}
|
||||
}
|
||||
response.Exchange = &structuredoutput.Exchange{PromptID: req.PromptID, Variables: frozen, Messages: convertMessages(details.Messages), FinalContent: result.RawOutput}
|
||||
if !result.Validation.IsValid {
|
||||
return response, classifiedFailure(parent, ctx, nil, result)
|
||||
}
|
||||
if decodeErr := json.Unmarshal([]byte(result.RawOutput), out); decodeErr != nil {
|
||||
var typeError *json.UnmarshalTypeError
|
||||
if errors.As(decodeErr, &typeError) {
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "structured output contains an unusable value", Cause: decodeErr}
|
||||
}
|
||||
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
|
||||
}
|
||||
if result.Exchange != nil {
|
||||
response.Exchange = &structuredoutput.Exchange{State: result.Exchange}
|
||||
}
|
||||
|
||||
response.Content = json.RawMessage(result.RawOutput)
|
||||
response.Provider = "openai-compatible"
|
||||
return response, nil
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (c *RoleClient) CompleteStructured(ctx context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
call, err := c.prepareCall(ctx, req, out, c.hooks.OnEvent)
|
||||
if err != nil {
|
||||
return structuredoutput.Response{}, err
|
||||
}
|
||||
response.Content = json.RawMessage(result.RawOutput)
|
||||
response.Provider = "openai-compatible"
|
||||
return response, nil
|
||||
result, err := c.runtime.Engine.Run(ctx, call.request)
|
||||
return call.finish(result, err)
|
||||
}
|
||||
|
||||
// OutputContract maps an Audita domain schema to the native request contract.
|
||||
@@ -231,6 +234,12 @@ func classifiedFailure(parent, local context.Context, cause error, result *pk.Ru
|
||||
failure.Cause = parent.Err()
|
||||
return failure
|
||||
}
|
||||
if errors.Is(cause, pk.ErrOperationTimeout) {
|
||||
failure.Class = structuredoutput.FailureRecoverable
|
||||
failure.Reason = outcomes.ReasonOperationTimeout
|
||||
failure.Message = "model operation timed out"
|
||||
return failure
|
||||
}
|
||||
if isSetupFailure(cause) {
|
||||
failure.Reason = outcomes.ReasonSetupFailure
|
||||
failure.Message = "structured model setup failed"
|
||||
@@ -298,10 +307,7 @@ func promptMetadata(details pk.PreparedRun) structuredoutput.PromptMetadata {
|
||||
}
|
||||
|
||||
func preparedSnapshot(details pk.PreparedRun) *structuredoutput.PreparedSnapshot {
|
||||
messages := make([]structuredoutput.Message, len(details.Messages))
|
||||
for i, message := range details.Messages {
|
||||
messages[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content}
|
||||
}
|
||||
messages := convertMessages(details.Messages)
|
||||
return &structuredoutput.PreparedSnapshot{Messages: messages, Prompt: promptMetadata(details)}
|
||||
}
|
||||
|
||||
|
||||
@@ -482,3 +482,14 @@ func TestPreparedAdapterUnknownValidationFailureRemainsFatal(t *testing.T) {
|
||||
t.Fatalf("unknown validation failure: %#v", failure)
|
||||
}
|
||||
}
|
||||
|
||||
// Preparation errors can carry the native operation deadline as their cause.
|
||||
// That deadline is recoverable even when the outer category is a setup phase.
|
||||
func TestOperationTimeoutDuringPreparationIsRecoverable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cause := errors.Join(pk.ErrPromptLoad, pk.ErrOperationTimeout, context.DeadlineExceeded)
|
||||
failure := classifiedFailure(ctx, ctx, cause, nil)
|
||||
if failure.Class != structuredoutput.FailureRecoverable || failure.Reason != outcomes.ReasonOperationTimeout || !errors.Is(failure, pk.ErrOperationTimeout) {
|
||||
t.Fatalf("failure = %#v", failure)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,141 +19,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNativeConversationReplayWithRepairContract(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": `profiles:
|
||||
- id: proposal-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: proposal-model
|
||||
- id: repair-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: repair-model
|
||||
prompts:
|
||||
- id: proposal-fixture
|
||||
messages:
|
||||
- role: system
|
||||
content: 'System {{.Subject}}'
|
||||
- role: user
|
||||
content: 'Proposal {{.Subject}}'
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: correction_set.v1.json
|
||||
- id: repair-fixture
|
||||
messages:
|
||||
- role: user
|
||||
content: 'Repair {{.Finding}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`,
|
||||
}
|
||||
for name, content := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var requests []struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct{ Role, Content string } `json:"messages"`
|
||||
ResponseFormat map[string]any `json:"response_format"`
|
||||
}
|
||||
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":""}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct{ Role, Content string } `json:"messages"`
|
||||
ResponseFormat map[string]any `json:"response_format"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
requests = append(requests, request)
|
||||
content, _ := json.Marshal(outputs[len(requests)-1])
|
||||
body := fmt.Sprintf(`{"model":%q,"choices":[{"message":{"content":%s}}]}`, request.Model, content)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
base := pk.RunRequest{PromptID: "proposal-fixture", ProfileID: "proposal-profile", Vars: map[string]string{"Subject": "source"}}
|
||||
original, err := runtime.Engine.PrepareExecution(ctx, base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prefix := original.Details().Messages
|
||||
initial, err := runtime.Engine.RunPrepared(ctx, original)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
suffix := []pk.RenderedMessage{{Role: pk.RoleAssistant, Content: initial.RawOutput}}
|
||||
contract, err := OutputContract(responseschema.CorrectionRepairKey, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, finding := range []string{"first", "second"} {
|
||||
instruction, err := runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: "repair-fixture", ProfileID: "repair-profile", Vars: map[string]string{"Finding": finding}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
suffix = append(suffix, instruction.Messages...)
|
||||
req := base
|
||||
req.ProfileID = "repair-profile"
|
||||
req.AppendedMessages = suffix
|
||||
req.Validation = contract
|
||||
handle, err := runtime.Engine.PrepareExecution(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
details := handle.Details()
|
||||
if !reflect.DeepEqual(details.Messages[:len(prefix)], prefix) || !reflect.DeepEqual(details.Messages[len(prefix):], suffix) {
|
||||
t.Fatalf("prepared conversation = %#v", details.Messages)
|
||||
}
|
||||
result, err := runtime.Engine.RunPrepared(ctx, handle)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Validation.IsValid {
|
||||
t.Fatalf("invalid result: %+v", result.Validation)
|
||||
}
|
||||
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: result.RawOutput})
|
||||
}
|
||||
if len(requests) != 3 || requests[0].Model != "proposal-model" || requests[1].Model != "repair-model" || requests[2].Model != "repair-model" {
|
||||
t.Fatalf("models: %+v", requests)
|
||||
}
|
||||
for i, wantLen := range []int{2, 4, 6} {
|
||||
req := requests[i]
|
||||
if len(req.Messages) != wantLen {
|
||||
t.Fatalf("request %d messages: %+v", i, req.Messages)
|
||||
}
|
||||
for j, message := range req.Messages {
|
||||
if message.Role != string(append(prefix, suffix...)[j].Role) || message.Content != append(prefix, suffix...)[j].Content {
|
||||
t.Fatalf("request %d message %d: %+v", i, j, message)
|
||||
}
|
||||
}
|
||||
encoded, _ := json.Marshal(req.ResponseFormat)
|
||||
expected := "corrections"
|
||||
if i > 0 {
|
||||
expected = "proposal_ref"
|
||||
}
|
||||
if !strings.Contains(string(encoded), expected) {
|
||||
t.Fatalf("request %d contract: %s", i, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
|
||||
func TestRoleClientContinuesIndependentRepairConversations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
@@ -207,17 +75,13 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
|
||||
t.Fatalf("capture: %+v %v", initial, err)
|
||||
}
|
||||
vars["SectionJSON"] = "changed"
|
||||
replay := &structuredoutput.Replay{Exchange: initial.Exchange}
|
||||
exchange := initial.Exchange
|
||||
for _, finding := range []string{"first", "second"} {
|
||||
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, Replay: replay}, &map[string]any{})
|
||||
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, ContinueFrom: exchange}, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.RepairInstruction) != 1 {
|
||||
t.Fatalf("instruction: %+v", result)
|
||||
}
|
||||
replay.Turns = append(replay.Turns, result.RepairInstruction...)
|
||||
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: result.FinalContent})
|
||||
exchange = result.Exchange
|
||||
}
|
||||
if !reflect.DeepEqual(models, []string{"proposal", "repair", "repair"}) {
|
||||
t.Fatalf("models = %v", models)
|
||||
@@ -235,21 +99,18 @@ func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
|
||||
!strings.Contains(string(first.Content), `"text":"first"`) || !reflect.DeepEqual(first, messages[2][len(prefix)+1]) {
|
||||
t.Fatalf("cached instruction changed during replay: first=%s replay=%s", first.Content, messages[2][len(prefix)+1].Content)
|
||||
}
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, Replay: &structuredoutput.Replay{Exchange: initial.Exchange}}, &map[string]any{})
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, ContinueFrom: initial.Exchange}, &map[string]any{})
|
||||
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(string(messages[3][len(prefix)+1].Content), "independent") {
|
||||
t.Fatalf("independent branch: %v %+v", err, messages)
|
||||
}
|
||||
bad := *initial.Exchange
|
||||
bad.Messages = append([]structuredoutput.Message(nil), bad.Messages...)
|
||||
bad.Messages[0].Content = "changed"
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, Replay: &structuredoutput.Replay{Exchange: &bad}}, &map[string]any{})
|
||||
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, ContinueFrom: &structuredoutput.Exchange{}}, &map[string]any{})
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || len(messages) != 4 {
|
||||
t.Fatalf("prefix failure: %v, calls %d", err, len(messages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairReplayUsesReplacementAndCountsStructuralRepair(t *testing.T) {
|
||||
func TestRepairContinuationUsesReplacementAndCountsStructuralRepair(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
@@ -291,11 +152,11 @@ func TestRepairReplayUsesReplacementAndCountsStructuralRepair(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, Replay: &structuredoutput.Replay{Exchange: base.Exchange}}, &map[string]any{})
|
||||
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, ContinueFrom: base.Exchange}, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] || len(result.RepairInstruction) != 1 || result.RepairInstruction[0].Content != "Replace fixture" {
|
||||
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] {
|
||||
t.Fatalf("result=%+v calls=%d", result, len(requests))
|
||||
}
|
||||
}
|
||||
|
||||
63
internal/framework/llm/coordinate.go
Normal file
63
internal/framework/llm/coordinate.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
var _ structuredoutput.Coordinator = (*RoleClient)(nil)
|
||||
|
||||
// Coordinate shares request construction and finalization with ordinary calls.
|
||||
// PromptKit alone advances history and consumes the additional-turn budget.
|
||||
func (c *RoleClient) Coordinate(ctx context.Context, start *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
|
||||
var exchange *pk.Exchange
|
||||
if start != nil {
|
||||
exchange, _ = start.State.(*pk.Exchange)
|
||||
}
|
||||
var pending *pendingCall
|
||||
result, err := c.runtime.Engine.Coordinate(ctx, pk.CoordinateRequest{
|
||||
Start: exchange, AdditionalTurns: budget, Observer: c.hooks.OnEvent,
|
||||
Next: func(ctx context.Context, turn pk.ConversationTurn) (*pk.RunRequest, error) {
|
||||
var delivered structuredoutput.Turn
|
||||
if turn.Result != nil {
|
||||
response, failure := pending.finish(turn.Result, nil)
|
||||
pending = nil
|
||||
delivered = structuredoutput.Turn{Response: &response, Err: failure}
|
||||
}
|
||||
selected, err := next(ctx, delivered)
|
||||
if err != nil || selected == nil {
|
||||
return nil, err
|
||||
}
|
||||
// Exhaustion needs a requested turn, but must not allocate a logical call.
|
||||
if turn.RemainingTurns == 0 {
|
||||
return &pk.RunRequest{}, nil
|
||||
}
|
||||
var output json.RawMessage
|
||||
pending, err = c.prepareCall(ctx, *selected, &output, nil)
|
||||
if err != nil {
|
||||
_, domainErr := next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: err})
|
||||
if domainErr != nil {
|
||||
return nil, domainErr
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &pending.request, nil
|
||||
},
|
||||
})
|
||||
if pending != nil {
|
||||
// Execution errors and withheld responses do not enter Next. Finalize their
|
||||
// diagnostic evidence and let the domain record the terminal failure once.
|
||||
response, failure := pending.finish(result.Undelivered, err)
|
||||
_, domainErr := next(ctx, structuredoutput.Turn{Response: &response, Err: failure})
|
||||
if domainErr != nil {
|
||||
return false, domainErr
|
||||
}
|
||||
if structuredoutput.IsRecoverable(failure) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return result.Outcome == pk.CoordinationExhausted, err
|
||||
}
|
||||
141
internal/framework/llm/coordinate_test.go
Normal file
141
internal/framework/llm/coordinate_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestCoordinatorFinalTurnFailuresAndAccounting(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
reply reply
|
||||
again bool
|
||||
reason outcomes.ReasonCode
|
||||
}{
|
||||
{name: "last turn assessed", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}},
|
||||
{name: "exhaustion", reply: reply{200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, again: true},
|
||||
{name: "invalid structure", reply: reply{200, `{"wrong":true}`}, reason: outcomes.ReasonInvalidOutputExhausted},
|
||||
{name: "provider error", reply: reply{400, ""}, reason: outcomes.ReasonProviderFailed},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, tc.reply}, 0, 0, nil)
|
||||
collector := NewObservationCollector()
|
||||
client, err := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
req.CaptureExchange = true
|
||||
base, err := client.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deliveries := 0
|
||||
exhausted, err := client.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil {
|
||||
deliveries++
|
||||
if tc.reason != "" {
|
||||
var failure *structuredoutput.Failure
|
||||
if !errors.As(turn.Err, &failure) || failure.Reason != tc.reason {
|
||||
t.Fatalf("failure=%v", turn.Err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if turn.Err != nil || len(turn.Response.Content) == 0 {
|
||||
t.Fatalf("turn=%+v", turn)
|
||||
}
|
||||
if !tc.again {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
if err != nil || exhausted != tc.again || deliveries != 1 || count.Load() != 2 {
|
||||
t.Fatalf("exhausted=%v delivered=%d calls=%d err=%v", exhausted, deliveries, count.Load(), err)
|
||||
}
|
||||
snapshot := collector.Snapshot()
|
||||
summary := Summarize(snapshot)
|
||||
if len(snapshot.Calls) != 2 || summary.ProviderAttempts != 2 || !summary.TerminalCoverageComplete {
|
||||
t.Fatalf("summary=%+v calls=%+v", summary, snapshot.Calls)
|
||||
}
|
||||
for _, call := range snapshot.Calls {
|
||||
if call.Finished.IsZero() {
|
||||
t.Fatal("unfinished logical call")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorCanceledDeliveryRetainsEvidence(t *testing.T) {
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}}, 0, 0, nil)
|
||||
collector := NewObservationCollector()
|
||||
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
req.CaptureExchange = true
|
||||
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
hooks := collector.Hooks()
|
||||
hooks.OnEvent = func(e pk.ExecutionEvent) {
|
||||
if e.Kind == pk.ExecutionFinished {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
client, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, hooks)
|
||||
deliveries := 0
|
||||
_, err = client.Coordinate(ctx, base.Exchange, 1, func(_ context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil {
|
||||
deliveries++
|
||||
if !errors.Is(turn.Err, context.Canceled) || turn.Response.Prepared == nil || turn.Response.FinalContent == "" || len(turn.Response.Content) != 0 {
|
||||
t.Fatalf("canceled turn=%+v", turn)
|
||||
}
|
||||
return nil, turn.Err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) || deliveries != 1 {
|
||||
t.Fatalf("deliveries=%d err=%v", deliveries, err)
|
||||
}
|
||||
if got := Summarize(collector.Snapshot()); got.ProviderAttempts != 2 || !got.TerminalCoverageComplete {
|
||||
t.Fatalf("summary=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorReleasesAdmissionBeforeNestedValidation(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}, {200, `{"proposal_ref":1,"action":"withdraw","correction":null}`}, {200, `{"validations":[]}`}}, 0, 0, nil)
|
||||
collector := NewObservationCollector()
|
||||
proposal, _ := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
|
||||
repair, _ := runtime.NewRoleClient("repair", "base", runtime.Config.LLM, collector.Hooks())
|
||||
validation, _ := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, collector.Hooks())
|
||||
req := adapterRequest(responseschema.CorrectionSetKey)
|
||||
req.CaptureExchange = true
|
||||
base, err := proposal.CompleteStructured(context.Background(), req, &map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = repair.Coordinate(context.Background(), base.Exchange, 1, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil {
|
||||
if turn.Err != nil {
|
||||
return nil, turn.Err
|
||||
}
|
||||
_, err := validation.CompleteStructured(ctx, adapterRequest(responseschema.ValidatorDecisionSetKey), &map[string]any{})
|
||||
return nil, err
|
||||
}
|
||||
return &structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "{}"}}, nil
|
||||
})
|
||||
summary := Summarize(collector.Snapshot())
|
||||
if err != nil || count.Load() != 3 || summary.ProviderAttempts != 3 || summary.LogicalCalls != 3 || !summary.TerminalCoverageComplete {
|
||||
t.Fatalf("calls=%d summary=%+v err=%v", count.Load(), summary, err)
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ func TestRoleClientExecutionBudgetsAndFailureClasses(t *testing.T) {
|
||||
capturing := request
|
||||
capturing.CaptureExchange = true
|
||||
response, err := client.CompleteStructured(context.Background(), capturing, &result)
|
||||
if tc.success && (response.Exchange == nil || response.Exchange.FinalContent != tc.outputs[len(tc.outputs)-1]) {
|
||||
if tc.success && (response.Exchange == nil || response.FinalContent != tc.outputs[len(tc.outputs)-1]) {
|
||||
t.Fatalf("captured final response: %+v", response.Exchange)
|
||||
}
|
||||
if (err == nil) != tc.success || calls != tc.wantCalls {
|
||||
|
||||
@@ -64,21 +64,24 @@ type LogicalCall struct {
|
||||
}
|
||||
|
||||
type ObservationSnapshot struct {
|
||||
Calls []LogicalCall `json:"calls"`
|
||||
Events []ExecutionObservation `json:"events"`
|
||||
Group *pk.ExecutionGroupSummary `json:"-"`
|
||||
Calls []LogicalCall `json:"calls"`
|
||||
Events []ExecutionObservation `json:"events"`
|
||||
}
|
||||
|
||||
type ExecutionSummary struct {
|
||||
LogicalCalls uint64 `json:"logical_calls"`
|
||||
ProviderAttempts uint64 `json:"provider_attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
StructuralRepairs uint64 `json:"structural_repairs"`
|
||||
Usage Usage `json:"usage"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
LogicalCalls uint64 `json:"logical_calls"`
|
||||
ProviderAttempts uint64 `json:"provider_attempts"`
|
||||
TransientRetries uint64 `json:"transient_retries"`
|
||||
StructuralRepairs uint64 `json:"structural_repairs"`
|
||||
Usage Usage `json:"usage"`
|
||||
}
|
||||
|
||||
// ObservationCollector accepts short synchronous callbacks from parallel calls.
|
||||
// Snapshot should be taken after the runner has joined active section work.
|
||||
type ObservationCollector struct {
|
||||
group pk.ExecutionGroup
|
||||
mu sync.Mutex
|
||||
calls map[uint64]LogicalCall
|
||||
events []ExecutionObservation
|
||||
@@ -137,6 +140,7 @@ func (c *ObservationCollector) recordEvent(callID uint64, context structuredoutp
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.group.Observe(event)
|
||||
count := func(value pk.ObservedTokenCount) TokenCount {
|
||||
return TokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
|
||||
}
|
||||
@@ -176,52 +180,28 @@ func (c *ObservationCollector) Snapshot() ObservationSnapshot {
|
||||
}
|
||||
return snapshot.Events[i].Time.Before(snapshot.Events[j].Time)
|
||||
})
|
||||
group := c.group.Snapshot()
|
||||
snapshot.Group = &group
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// Summarize counts terminal usage once per logical call. PromptKit's terminal
|
||||
// usage already aggregates its attempts, retries, and repairs.
|
||||
// Summarize delegates deduplication and terminal accounting to PromptKit. A
|
||||
// filtered diagnostic view selects its own group; overlapping views are not additive.
|
||||
func Summarize(snapshot ObservationSnapshot) ExecutionSummary {
|
||||
result := ExecutionSummary{LogicalCalls: uint64(len(snapshot.Calls))}
|
||||
terminal := make(map[uint64]ExecutionObservation)
|
||||
for _, event := range snapshot.Events {
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
terminal[event.CallID] = event
|
||||
summary := snapshot.Group
|
||||
if summary == nil {
|
||||
var group pk.ExecutionGroup
|
||||
count := func(c TokenCount) pk.ObservedTokenCount {
|
||||
return pk.ObservedTokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
}
|
||||
dispatched := false
|
||||
aggregate := func(dst *TokenCount, source TokenCount) {
|
||||
if source.Known {
|
||||
dst.Value += source.Value
|
||||
dst.Known = true
|
||||
for _, event := range snapshot.Events {
|
||||
group.Observe(pk.ExecutionEvent{Kind: pk.ExecutionEventKind(event.Kind), RunID: event.RunID, Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.StructuralRepairs, Usage: pk.ObservedTokenUsage{PromptTokens: count(event.Usage.Prompt), CompletionTokens: count(event.Usage.Completion), TotalTokens: count(event.Usage.Total), CachedTokens: count(event.Usage.Cached), CacheWriteTokens: count(event.Usage.CacheWrite)}})
|
||||
}
|
||||
dst.Complete = dst.Complete && source.Complete
|
||||
value := group.Snapshot()
|
||||
summary = &value
|
||||
}
|
||||
result.Usage = Usage{Prompt: TokenCount{Complete: true}, Completion: TokenCount{Complete: true}, Total: TokenCount{Complete: true}, Cached: TokenCount{Complete: true}, CacheWrite: TokenCount{Complete: true}}
|
||||
for _, call := range snapshot.Calls {
|
||||
event, ok := terminal[call.CallID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
result.ProviderAttempts += event.Attempts
|
||||
result.TransientRetries += event.TransientRetries
|
||||
result.StructuralRepairs += event.StructuralRepairs
|
||||
if event.Attempts == 0 {
|
||||
continue
|
||||
}
|
||||
dispatched = true
|
||||
aggregate(&result.Usage.Prompt, event.Usage.Prompt)
|
||||
aggregate(&result.Usage.Completion, event.Usage.Completion)
|
||||
aggregate(&result.Usage.Total, event.Usage.Total)
|
||||
aggregate(&result.Usage.Cached, event.Usage.Cached)
|
||||
aggregate(&result.Usage.CacheWrite, event.Usage.CacheWrite)
|
||||
count := func(c pk.ObservedTokenCount) TokenCount {
|
||||
return TokenCount{Value: c.Value, Known: c.Known, Complete: c.Complete}
|
||||
}
|
||||
if !dispatched {
|
||||
result.Usage.Prompt.Known = true
|
||||
result.Usage.Completion.Known = true
|
||||
result.Usage.Total.Known = true
|
||||
result.Usage.Cached.Known = true
|
||||
result.Usage.CacheWrite.Known = true
|
||||
}
|
||||
return result
|
||||
return ExecutionSummary{LogicalCalls: uint64(len(snapshot.Calls)), TerminalCoverageComplete: summary.TerminalCoverageComplete, ProviderAttempts: summary.Attempts, TransientRetries: summary.TransientRetries, StructuralRepairs: summary.RepairCalls, Usage: Usage{Prompt: count(summary.Usage.PromptTokens), Completion: count(summary.Usage.CompletionTokens), Total: count(summary.Usage.TotalTokens), Cached: count(summary.Usage.CachedTokens), CacheWrite: count(summary.Usage.CacheWriteTokens)}}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package llm
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -23,8 +24,8 @@ func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
|
||||
}
|
||||
call(1) // preparation failed before PromptKit emitted events
|
||||
call(2) // capacity rejected without a dispatched attempt
|
||||
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{Kind: pk.ExecutionAttempt, Dispatched: false})
|
||||
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 0})
|
||||
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{RunID: "run-2", Kind: pk.ExecutionAttempt, Dispatched: false})
|
||||
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{Role: "proposal"}, pk.ExecutionEvent{RunID: "run-2", Kind: pk.ExecutionFinished, Attempts: 0, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CompletionTokens: pk.ObservedTokenCount{Known: true, Complete: true}, TotalTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CachedTokens: pk.ObservedTokenCount{Known: true, Complete: true}, CacheWriteTokens: pk.ObservedTokenCount{Known: true, Complete: true}}})
|
||||
summary := Summarize(collector.Snapshot())
|
||||
if summary.LogicalCalls != 2 || summary.ProviderAttempts != 0 || summary.Usage.Total != (TokenCount{Known: true, Complete: true}) {
|
||||
t.Fatalf("zero-attempt accounting: %+v", summary)
|
||||
@@ -32,16 +33,16 @@ func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
|
||||
|
||||
call(3)
|
||||
context := structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar", ValidatorName: "editorial_review"}
|
||||
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{Kind: pk.ExecutionAttempt, Dispatched: true, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 100, Known: true, Complete: true}}})
|
||||
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 3, TransientRetries: 1, RepairCalls: 1, Usage: pk.ObservedTokenUsage{
|
||||
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{RunID: "run-3", Kind: pk.ExecutionAttempt, Dispatched: true, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 100, Known: true, Complete: true}}})
|
||||
hooks.OnEventWithCall(3, context, pk.ExecutionEvent{RunID: "run-3", Kind: pk.ExecutionFinished, Attempts: 3, TransientRetries: 1, RepairCalls: 1, Usage: pk.ObservedTokenUsage{
|
||||
PromptTokens: pk.ObservedTokenCount{Value: 5, Known: true, Complete: false},
|
||||
TotalTokens: pk.ObservedTokenCount{Value: 8, Known: true, Complete: false},
|
||||
}})
|
||||
call(4)
|
||||
hooks.OnEventWithCall(4, context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: false, Complete: false}, TotalTokens: pk.ObservedTokenCount{Known: false, Complete: false}}})
|
||||
hooks.OnEventWithCall(4, context, pk.ExecutionEvent{RunID: "run-4", Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{PromptTokens: pk.ObservedTokenCount{Known: false, Complete: false}, TotalTokens: pk.ObservedTokenCount{Known: false, Complete: false}}})
|
||||
snapshot := collector.Snapshot()
|
||||
summary = Summarize(snapshot)
|
||||
if summary.LogicalCalls != 4 || summary.ProviderAttempts != 4 || summary.TransientRetries != 1 || summary.StructuralRepairs != 1 || summary.Usage.Total.Value != 8 || !summary.Usage.Total.Known || summary.Usage.Total.Complete || summary.Usage.Prompt.Value != 5 || !summary.Usage.Prompt.Known || summary.Usage.Prompt.Complete || summary.Usage.Cached.Known || summary.Usage.Cached.Complete {
|
||||
if summary.LogicalCalls != 4 || summary.ProviderAttempts != 4 || summary.TransientRetries != 1 || summary.StructuralRepairs != 1 || summary.Usage.Total.Value != 8 || !summary.Usage.Total.Known || summary.Usage.Total.Complete || summary.Usage.Prompt.Value != 5 || !summary.Usage.Prompt.Known || summary.Usage.Prompt.Complete || !summary.Usage.Cached.Known || summary.Usage.Cached.Complete {
|
||||
t.Fatalf("partial usage or double count: %+v", summary)
|
||||
}
|
||||
raw, err := json.Marshal(snapshot)
|
||||
@@ -68,7 +69,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T)
|
||||
context := structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar", SectionIndex: &index}
|
||||
started := time.Now().UTC()
|
||||
hooks.OnCall(structuredoutput.CallRecord{CallID: uint64(id), Context: context, Started: started})
|
||||
hooks.OnEventWithCall(uint64(id), context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, RunID: "run", Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 1, Known: true, Complete: true}}})
|
||||
hooks.OnEventWithCall(uint64(id), context, pk.ExecutionEvent{Kind: pk.ExecutionFinished, RunID: fmt.Sprint(id), Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 1, Known: true, Complete: true}}})
|
||||
hooks.OnCall(structuredoutput.CallRecord{CallID: uint64(id), Context: context, Started: started, Finished: time.Now().UTC()})
|
||||
index = -1
|
||||
}(i)
|
||||
@@ -124,3 +125,18 @@ func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) {
|
||||
t.Fatalf("logical failures: %+v", snapshot.Calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationCoverageIsIndependentOfUsage(t *testing.T) {
|
||||
collector := NewObservationCollector()
|
||||
hooks := collector.Hooks()
|
||||
hooks.OnEventWithCall(1, structuredoutput.DiagnosticContext{}, pk.ExecutionEvent{RunID: "pending", Kind: pk.ExecutionAttempt})
|
||||
got := Summarize(collector.Snapshot())
|
||||
if got.TerminalCoverageComplete || got.Usage.Total.Known || got.Usage.Total.Complete {
|
||||
t.Fatalf("incomplete group=%+v", got)
|
||||
}
|
||||
hooks.OnEventWithCall(2, structuredoutput.DiagnosticContext{}, pk.ExecutionEvent{RunID: "finished", Kind: pk.ExecutionFinished, Attempts: 1, Usage: pk.ObservedTokenUsage{TotalTokens: pk.ObservedTokenCount{Value: 7, Known: true, Complete: true}}})
|
||||
got = Summarize(collector.Snapshot())
|
||||
if got.TerminalCoverageComplete || !got.Usage.Total.Known || !got.Usage.Total.Complete || got.Usage.Total.Value != 7 {
|
||||
t.Fatalf("partial coverage=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,15 +20,16 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
TerminalCoverageComplete bool `json:"terminal_coverage_complete"`
|
||||
RequestedPromptID string `json:"requested_prompt_id,omitempty"`
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
Prompt structuredoutput.PromptMetadata `json:"prompt"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
ElapsedMS int64 `json:"elapsed_ms"`
|
||||
Failure structuredoutput.FailureKind `json:"failure,omitempty"`
|
||||
Summary reporting.LLMSummary `json:"summary"`
|
||||
}
|
||||
|
||||
type executionEvent struct {
|
||||
@@ -72,7 +73,7 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
value := call.Finished
|
||||
finished = &value
|
||||
}
|
||||
calls = append(calls, executionCall{RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
calls = append(calls, executionCall{TerminalCoverageComplete: llm.Summarize(group).TerminalCoverageComplete, RequestedPromptID: call.RequestedPromptID, CallID: call.CallID, Context: call.Context, Prompt: call.Prompt, Started: call.Started, Finished: finished, ElapsedMS: call.WallDuration.Milliseconds(), Failure: call.Failure, Summary: mapLLMSummary(llm.Summarize(group))})
|
||||
}
|
||||
for _, event := range snapshot.Events {
|
||||
var generation, attempt *uint64
|
||||
@@ -87,7 +88,8 @@ func BuildLLMExecution(snapshot llm.ObservationSnapshot) any {
|
||||
events = append(events, executionEvent{CallID: event.CallID, Context: event.Context, Kind: event.Kind, RunID: event.RunID, Time: event.Time, BackendID: event.BackendID, ProfileID: event.ProfileID, Model: event.Model, Generation: generation, Attempt: attempt, Dispatched: event.Dispatched, CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(), Outcome: event.Outcome, Failure: event.Failure, StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted, Attempts: event.Attempts, TransientRetries: event.TransientRetries, StructuralRepairs: event.StructuralRepairs, Usage: mapLLMSummary(llm.ExecutionSummary{Usage: event.Usage}).Usage})
|
||||
}
|
||||
return struct {
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{calls, events}
|
||||
Summary llm.ExecutionSummary `json:"summary"`
|
||||
Calls []executionCall `json:"calls"`
|
||||
Events []executionEvent `json:"events"`
|
||||
}{llm.Summarize(snapshot), calls, events}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
|
||||
{CallID: 1, Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleInstance: "grammar-1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "primary", ProfileID: "draft", Model: "m"}, Started: start, Finished: start.Add(1500 * time.Millisecond), WallDuration: 1500 * time.Millisecond},
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation"}, Prompt: structuredoutput.PromptMetadata{BackendID: "secondary"}, Started: start, Finished: start.Add(5 * time.Millisecond), WallDuration: 5 * time.Millisecond, Failure: structuredoutput.FailureRuntime},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
if err != nil {
|
||||
@@ -46,7 +46,7 @@ func BenchmarkExecutionReports(b *testing.B) {
|
||||
snapshot := llm.ObservationSnapshot{}
|
||||
for i := 0; i < n; i++ {
|
||||
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{RunID: fmt.Sprint(i), CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
}
|
||||
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
@@ -64,9 +64,9 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
|
||||
{CallID: 2, Context: structuredoutput.DiagnosticContext{Role: "validation", ModuleInstance: "grammar_1"}, Prompt: structuredoutput.PromptMetadata{BackendID: "a"}},
|
||||
},
|
||||
Events: []llm.ExecutionObservation{
|
||||
{CallID: 1, Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
|
||||
{CallID: 3, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 2, Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 1, RunID: "run-1", Kind: string(pk.ExecutionFinished), Attempts: 1, Usage: llm.Usage{Total: llm.TokenCount{Value: 7, Known: true, Complete: true}}},
|
||||
{CallID: 3, RunID: "run-3", Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
{CallID: 2, RunID: "run-2", Kind: string(pk.ExecutionFinished), Attempts: 1},
|
||||
},
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
)
|
||||
|
||||
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
|
||||
exchange := &structuredoutput.Exchange{PromptID: "captured", FinalContent: "response"}
|
||||
exchange := &structuredoutput.Exchange{}
|
||||
client := &fakeStructuredClient{
|
||||
completion: structuredoutput.Response{Exchange: exchange},
|
||||
responses: []StructuredCorrectionSet{
|
||||
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
@@ -117,8 +118,11 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
return findings
|
||||
}
|
||||
findings := safeFindings(rejected)
|
||||
replay := &structuredoutput.Replay{Exchange: input.Exchange}
|
||||
for attempt := 0; attempt < input.Spec.Settings.LLM.CorrectionRepairAttempts; attempt++ {
|
||||
coordinator, ok := input.Spec.RepairClient.(structuredoutput.Coordinator)
|
||||
if !ok {
|
||||
return finish(fmt.Errorf("correction repair coordinator is unavailable"))
|
||||
}
|
||||
buildRequest := func() (*structuredoutput.Request, error) {
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: position + 1,
|
||||
Correction: proposal_generation.StructuredCorrectionProposal{SegmentReference: localRef, OriginalText: current.OriginalText, CorrectedText: current.CorrectedText},
|
||||
@@ -134,12 +138,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
payload, err := request.JSON()
|
||||
if err != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
|
||||
}
|
||||
exhausted, coordinationErr := coordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response == nil {
|
||||
return buildRequest()
|
||||
}
|
||||
completion, callErr := *turn.Response, turn.Err
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
|
||||
var response proposal_generation.RepairResponse
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||
completion, callErr := input.Spec.RepairClient.CompleteStructured(ctx, structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Replay: replay, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, &response)
|
||||
if callErr == nil {
|
||||
_ = json.Unmarshal(completion.Content, &response)
|
||||
}
|
||||
|
||||
operation := outcomes.Operation{
|
||||
Reference: outcomes.Reference{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID, ModuleKey: input.Spec.ModuleKey,
|
||||
@@ -183,26 +198,23 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
finishAttempt(callErr, nil, validateSectionCandidatesResult{})
|
||||
if structuredoutput.IsRecoverable(callErr) {
|
||||
branch.Outcome = "unavailable"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
branches = append(branches, branch)
|
||||
return finish(callErr)
|
||||
return nil, callErr
|
||||
}
|
||||
decoded, err := proposal_generation.DecodeRepairResponse(completion.Content, position+1)
|
||||
if err != nil {
|
||||
operation.FailureReason = outcomes.ReasonInvalidOutput
|
||||
finishAttempt(err, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "unavailable"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
operation.Succeeded = true
|
||||
operation.RecoveryReasons = outcomes.ExecutionRecoveries(operation.Counts)
|
||||
replay.Turns = append(replay.Turns, completion.RepairInstruction...)
|
||||
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: completion.FinalContent})
|
||||
if decoded.Action == "withdraw" {
|
||||
finishAttempt(nil, nil, validateSectionCandidatesResult{})
|
||||
branch.Outcome = "withdrawn"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
revision := decoded.Correction
|
||||
localRef = revision.SegmentReference
|
||||
@@ -225,7 +237,7 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
operation.RecoveryReasons = append(operation.RecoveryReasons, outcomes.RecoveryProposalsFiltered)
|
||||
}
|
||||
for i := range validated.operations {
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attempt+1
|
||||
proposalIndex, attemptNumber := original.ProposalIndex, attemptNumber
|
||||
validated.operations[i].Reference.ProposalIndex = &proposalIndex
|
||||
validated.operations[i].Reference.RepairAttempt = &attemptNumber
|
||||
}
|
||||
@@ -236,32 +248,35 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
finishAttempt(validationErr, ¤t, validated)
|
||||
initial.operations = append(initial.operations, validated.operations...)
|
||||
if validationErr != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(validationErr)
|
||||
return nil, validationErr
|
||||
}
|
||||
if approved {
|
||||
branch.Outcome = "approved"
|
||||
initial.approved = append(initial.approved, current)
|
||||
superseded[original.ProposalIndex] = true
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
if actionable(original.ProposalIndex, validated.rejected) {
|
||||
if attempt+1 == input.Spec.Settings.LLM.CorrectionRepairAttempts {
|
||||
branch.Outcome = "exhausted"
|
||||
break
|
||||
}
|
||||
findings = safeFindings(validated.rejected)
|
||||
continue
|
||||
return buildRequest()
|
||||
}
|
||||
branch.Outcome = "unavailable"
|
||||
for _, d := range validated.rejected {
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
branch.Outcome = "rejected"
|
||||
break
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
break
|
||||
return nil, nil
|
||||
})
|
||||
if exhausted {
|
||||
branch.Outcome = "exhausted"
|
||||
}
|
||||
if coordinationErr != nil {
|
||||
branches = append(branches, branch)
|
||||
return finish(coordinationErr)
|
||||
}
|
||||
|
||||
branches = append(branches, branch)
|
||||
}
|
||||
return finish(nil)
|
||||
|
||||
@@ -4,10 +4,17 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
@@ -20,6 +27,7 @@ import (
|
||||
)
|
||||
|
||||
type repairClient struct {
|
||||
t *testing.T
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
@@ -27,11 +35,7 @@ type repairClient struct {
|
||||
}
|
||||
|
||||
func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutput.Request, out any) (structuredoutput.Response, error) {
|
||||
if req.Replay != nil {
|
||||
copyReplay := *req.Replay
|
||||
copyReplay.Turns = append([]structuredoutput.Message(nil), req.Replay.Turns...)
|
||||
req.Replay = ©Replay
|
||||
}
|
||||
|
||||
c.calls = append(c.calls, req)
|
||||
if c.err != nil {
|
||||
return structuredoutput.Response{}, c.err
|
||||
@@ -40,18 +44,18 @@ func (c *repairClient) CompleteStructured(_ context.Context, req structuredoutpu
|
||||
if err := json.Unmarshal([]byte(value), out); err != nil {
|
||||
return structuredoutput.Response{}, err
|
||||
}
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, RepairInstruction: []structuredoutput.Message{{Role: "user", Content: "instruction"}}}, nil
|
||||
return structuredoutput.Response{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: "synthetic-secret"}}}, Content: json.RawMessage(value), FinalContent: value, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}
|
||||
|
||||
func (c *repairClient) ConfiguredSecrets(context.Context) []string { return c.secrets }
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal", FinalContent: "original response"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -85,7 +89,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
input := validateSectionCandidatesInput{
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
@@ -93,7 +97,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -125,7 +129,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{
|
||||
client := &repairClient{t: t, outputs: []string{
|
||||
`{"proposal_ref":1,"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}`,
|
||||
`{"proposal_ref":1,"action":"withdraw","correction":null}`,
|
||||
}}
|
||||
@@ -138,7 +142,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{PromptID: "proposal"},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -175,9 +179,9 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed"}, 1, "unavailable", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{outputs: tc.outputs, err: tc.err}
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -189,8 +193,8 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
|
||||
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
|
||||
}
|
||||
if tc.calls == 2 && len(client.calls[1].Replay.Turns) != 2 {
|
||||
t.Fatalf("history: %+v", client.calls[1].Replay.Turns)
|
||||
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
|
||||
t.Fatalf("history: %+v", client.calls[1].ContinueFrom)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -202,11 +206,11 @@ func (m repairModule) RepairPromptID() string { return "repair" }
|
||||
|
||||
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}`,
|
||||
`{"proposal_ref":3,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}, nil
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
|
||||
@@ -246,11 +250,11 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}`}}
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -271,9 +275,9 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}
|
||||
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -285,12 +289,12 @@ func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
client := &repairClient{secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, secrets: []string{"synthetic-secret"}, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":2,"action":"withdraw","correction":null}`}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -318,14 +322,14 @@ func TestRepairArtifactsAreUniqueAndRedacted(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
path := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
warnings := 0
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: path, ReportDiagnosticError: func(error) { warnings++ }}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -337,7 +341,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
client := &repairClient{}
|
||||
client := &repairClient{t: t}
|
||||
for _, rejected := range [][]ValidatorRejectedChange{
|
||||
{{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}},
|
||||
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
|
||||
@@ -352,10 +356,10 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRepairFeedbackRedactsValidationCredentials(t *testing.T) {
|
||||
client := &repairClient{outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
validation := &repairClient{secrets: []string{"validation-secret"}}
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"action":"withdraw","correction":null}`}}
|
||||
validation := &repairClient{t: t, secrets: []string{"validation-secret"}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{PromptID: "proposal"}, DiagnosticsDir: t.TempDir()}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, ValidationLLMClient: validation, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, DiagnosticsDir: t.TempDir()}
|
||||
initial := validateSectionCandidatesResult{rejected: []ValidatorRejectedChange{{ProposalIndex: 0, ValidatorName: "editorial_review", Disposition: validators.DispositionRejected, ReasonCode: "rejected", Message: "model echoed validation-secret"}}}
|
||||
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
|
||||
if err != nil {
|
||||
@@ -393,3 +397,66 @@ func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
type repairTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f repairTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
// Exercise the real adapter and upstream coordinator; only model responses are scripted.
|
||||
func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
|
||||
t := c.t
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: 'synthetic-secret {{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var selected *structuredoutput.Request
|
||||
initial := true
|
||||
runtime, err := llm.NewRuntime(ctx, loaded, &http.Client{Transport: repairTransport(func(*http.Request) (*http.Response, error) {
|
||||
content := `{"corrections":[]}`
|
||||
if initial {
|
||||
initial = false
|
||||
} else {
|
||||
c.calls = append(c.calls, *selected)
|
||||
content = c.outputs[len(c.calls)-1]
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}})
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body)))}, nil
|
||||
})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("repair", "fixture", config.LLM{OperationTimeout: time.Second}, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var output map[string]any
|
||||
base, err := client.CompleteStructured(ctx, structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, CaptureExchange: true, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}, &output)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return client.Coordinate(ctx, base.Exchange, budget, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
req, err := next(ctx, turn)
|
||||
if req != nil && err == nil {
|
||||
selected = req
|
||||
if c.err != nil {
|
||||
c.calls = append(c.calls, *req)
|
||||
_, err = next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: c.err})
|
||||
return nil, err
|
||||
}
|
||||
// Test prompt IDs vary; all use the same supplied-data fixture definition.
|
||||
copy := *req
|
||||
copy.PromptID = "repair"
|
||||
return ©, nil
|
||||
}
|
||||
return req, err
|
||||
})
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ type Request struct {
|
||||
Schema responseschema.Key `json:"schema"`
|
||||
Context DiagnosticContext `json:"context"`
|
||||
CaptureExchange bool `json:"-"`
|
||||
Replay *Replay `json:"-"`
|
||||
ContinueFrom *Exchange `json:"-"`
|
||||
}
|
||||
|
||||
// PromptMetadata is the execution snapshot returned by the runtime when available.
|
||||
@@ -44,33 +44,23 @@ type PromptMetadata struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
}
|
||||
|
||||
// Exchange holds a section's original prepared conversation for read-only replay.
|
||||
// Exchange is an opaque immutable conversation owned by the runtime adapter.
|
||||
type Exchange struct {
|
||||
PromptID string
|
||||
Variables map[string]string
|
||||
Messages []Message
|
||||
FinalContent string
|
||||
}
|
||||
|
||||
// Replay carries a section exchange and the prior turns of one repair branch.
|
||||
type Replay struct {
|
||||
Exchange *Exchange
|
||||
Turns []Message
|
||||
State any
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Exchange *Exchange `json:"-"`
|
||||
RepairInstruction []Message `json:"-"`
|
||||
Execution outcomes.ExecutionCounts `json:"execution"`
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
Exchange *Exchange `json:"-"`
|
||||
Execution outcomes.ExecutionCounts `json:"execution"`
|
||||
Content json.RawMessage `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Prompt PromptMetadata `json:"prompt"`
|
||||
Prepared *PreparedSnapshot `json:"prepared,omitempty"`
|
||||
FinalContent string `json:"final_content,omitempty"`
|
||||
}
|
||||
|
||||
// PreparedSnapshot is the exact rendered request snapshot. Its messages may
|
||||
@@ -108,3 +98,16 @@ type CallRecord struct {
|
||||
type Client interface {
|
||||
CompleteStructured(ctx context.Context, req Request, out any) (Response, error)
|
||||
}
|
||||
|
||||
// Turn delivers a completed correction request, or its failure. The initial
|
||||
// callback has no response. A failure is terminal for this coordination.
|
||||
type Turn struct {
|
||||
Response *Response
|
||||
Err error
|
||||
}
|
||||
|
||||
// Coordinator delegates bounded conversation progression to the runtime.
|
||||
// Returning no request stops the branch; requesting beyond the budget exhausts it.
|
||||
type Coordinator interface {
|
||||
Coordinate(context.Context, *Exchange, int, func(context.Context, Turn) (*Request, error)) (bool, error)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user