eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Carry proposal exchanges into repair replay

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:18:05 +00:00
parent eb45d0b623
commit 00848203d7
10 changed files with 267 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/framework/llm/adapter.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"sync"
"time"
@@ -72,6 +73,7 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
var record structuredoutput.CallRecord
if c.hooks.OnCall != nil || c.hooks.OnEventWithCall != nil {
record.CallID = c.runtime.nextCallID.Add(1)
record.RequestedPromptID = req.PromptID
record.Context = copyContext(req.Context)
record.Context.Role = c.role
}
@@ -106,8 +108,27 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
for name, value := range prompts.SharedVariables() {
vars[name] = value
}
baseID, baseVars := req.PromptID, vars
var suffix []pk.RenderedMessage
if req.Replay != nil {
if req.Replay.Exchange == nil {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "repair exchange is unavailable"}
}
instruction, renderErr := c.runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars})
if renderErr != nil {
return response, classifiedFailure(parent, ctx, renderErr, nil)
}
response.RepairInstruction = convertMessages(instruction.Messages)
exchange := req.Replay.Exchange
baseID, baseVars = exchange.PromptID, exchange.Variables
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: exchange.FinalContent})
for _, message := range req.Replay.Turns {
suffix = append(suffix, pk.RenderedMessage{Role: message.Role, Content: message.Content})
}
suffix = append(suffix, instruction.Messages...)
}
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
PromptID: baseID, ProfileID: c.profileID, Vars: baseVars, AppendedMessages: suffix,
TransientRetries: &c.settings.TransientRetries,
Validation: contract,
Observer: func(event pk.ExecutionEvent) {
@@ -130,6 +151,12 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
}
defer handle.Discard()
details := handle.Details()
if req.Replay != nil {
prefix := req.Replay.Exchange.Messages
if len(details.Messages) < len(prefix) || !reflect.DeepEqual(convertMessages(details.Messages[:len(prefix)]), prefix) {
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "repair conversation prefix changed"}
}
}
if name := details.EffectiveModelParams.APIKeyEnv; name != "" {
c.credentialMu.Lock()
if c.credentialNames == nil {
@@ -168,6 +195,13 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
}
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonInternalFailure, Message: "validated structured output could not be decoded", Cause: decodeErr}
}
if req.CaptureExchange && req.Replay == nil {
frozen := make(map[string]string, len(vars))
for key, value := range vars {
frozen[key] = value
}
response.Exchange = &structuredoutput.Exchange{PromptID: req.PromptID, Variables: frozen, Messages: convertMessages(details.Messages), FinalContent: result.RawOutput}
}
response.Content = json.RawMessage(result.RawOutput)
response.Provider = "openai-compatible"
return response, nil
@@ -277,3 +311,11 @@ func (c *RoleClient) ConfiguredSecrets(ctx context.Context) []string {
c.credentialMu.Unlock()
return c.runtime.ConfiguredSecrets(ctx, names...)
}
func convertMessages(messages []pk.RenderedMessage) []structuredoutput.Message {
result := make([]structuredoutput.Message, len(messages))
for i, message := range messages {
result[i] = structuredoutput.Message{Role: string(message.Role), Content: message.Content}
}
return result
}

141
internal/framework/llm/conversation_test.go
View File

@@ -3,6 +3,7 @@ package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -11,9 +12,12 @@ import (
"reflect"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
@@ -148,3 +152,140 @@ prompts:
}
}
}
func TestRoleClientCapturesAndReplaysRepairConversation(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: proposal\n endpoint: https://example.invalid/v1\n model: proposal\n - id: repair\n endpoint: https://example.invalid/v1\n model: repair\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
var messages [][]structuredoutput.Message
var models []string
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"a","corrected_text":"b","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var req struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
models = append(models, req.Model)
messages = append(messages, req.Messages)
content, _ := json.Marshal(outputs[len(messages)-1])
body := fmt.Sprintf(`{"model":%q,"choices":[{"message":{"content":%s}}]}`, req.Model, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
settings := config.LLM{OperationTimeout: 10 * time.Second}
proposal, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
repair, err := runtime.NewRoleClient("repair", "repair", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
vars := map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}
initial, err := proposal.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: vars, CaptureExchange: true}, &map[string]any{})
if err != nil || initial.Exchange == nil {
t.Fatalf("capture: %+v %v", initial, err)
}
vars["SectionJSON"] = "changed"
replay := &structuredoutput.Replay{Exchange: initial.Exchange}
for _, finding := range []string{"first", "second"} {
result, err := repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": finding}, Replay: replay}, &map[string]any{})
if err != nil {
t.Fatal(err)
}
if len(result.RepairInstruction) != 1 {
t.Fatalf("instruction: %+v", result)
}
replay.Turns = append(replay.Turns, result.RepairInstruction...)
replay.Turns = append(replay.Turns, structuredoutput.Message{Role: "assistant", Content: result.FinalContent})
}
if !reflect.DeepEqual(models, []string{"proposal", "repair", "repair"}) {
t.Fatalf("models = %v", models)
}
prefix := initial.Exchange.Messages
if len(messages) != 3 || !reflect.DeepEqual(messages[0], prefix) || !reflect.DeepEqual(messages[1][:len(prefix)], prefix) || !reflect.DeepEqual(messages[2][:len(prefix)], prefix) {
t.Fatalf("prefixes: %+v", messages)
}
if len(messages[1]) != len(prefix)+2 || len(messages[2]) != len(prefix)+4 || messages[2][len(prefix)+2].Content != outputs[1] {
t.Fatalf("turns: %+v", messages)
}
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "independent"}, Replay: &structuredoutput.Replay{Exchange: initial.Exchange}}, &map[string]any{})
if err != nil || len(messages) != 4 || len(messages[3]) != len(prefix)+2 || !strings.Contains(messages[3][len(prefix)+1].Content, "independent") {
t.Fatalf("independent branch: %v %+v", err, messages)
}
bad := *initial.Exchange
bad.Messages = append([]structuredoutput.Message(nil), bad.Messages...)
bad.Messages[0].Content = "changed"
_, err = repair.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "failure"}, Replay: &structuredoutput.Replay{Exchange: &bad}}, &map[string]any{})
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != structuredoutput.FailureFatal || len(messages) != 4 {
t.Fatalf("prefix failure: %v, calls %d", err, len(messages))
}
}
func TestRepairReplayUsesReplacementAndCountsStructuralRepair(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\ncollections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\n",
"repair.txt": "Replace {{.RepairRequestJSON}}",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
outputs := []string{`{"corrections":[]}`, `{"bad":true}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
var requests [][]structuredoutput.Message
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
var req struct {
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatal(err)
}
requests = append(requests, req.Messages)
content, _ := json.Marshal(outputs[len(requests)-1])
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("repair", "fixture", config.LLM{OperationTimeout: 10 * time.Second, RepairAttempts: 1}, CallHooks{})
if err != nil {
t.Fatal(err)
}
base, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, CaptureExchange: true}, &map[string]any{})
if err != nil {
t.Fatal(err)
}
result, err := client.CompleteStructured(context.Background(), structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarRepair, Schema: responseschema.CorrectionRepairKey, Variables: map[string]string{"RepairRequestJSON": "fixture"}, Replay: &structuredoutput.Replay{Exchange: base.Exchange}}, &map[string]any{})
if err != nil {
t.Fatal(err)
}
if len(requests) != 3 || result.Execution.ProviderAttempts != 2 || result.Execution.StructuralRepairs != 1 || result.FinalContent != outputs[2] || len(result.RepairInstruction) != 1 || result.RepairInstruction[0].Content != "Replace fixture" {
t.Fatalf("result=%+v calls=%d", result, len(requests))
}
}