Define correction repair response and conversation replay
This commit is contained in:
@@ -15,6 +15,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
|
|||||||
|
|
||||||
## Stage 1 — Define the repair response and prove native conversation composition
|
## Stage 1 — Define the repair response and prove native conversation composition
|
||||||
|
|
||||||
|
**Completed.**
|
||||||
|
|
||||||
1. In `internal/framework/responseschema`, register `CorrectionRepairKey` with
|
1. In `internal/framework/responseschema`, register `CorrectionRepairKey` with
|
||||||
schema ID `audita.correction_repair`, name `audita_correction_repair_v1`, and
|
schema ID `audita.correction_repair`, name `audita_correction_repair_v1`, and
|
||||||
asset `correction_repair.v1.json`. Use a closed JSON object with three
|
asset `correction_repair.v1.json`. Use a closed JSON object with three
|
||||||
|
|||||||
150
internal/framework/llm/conversation_test.go
Normal file
150
internal/framework/llm/conversation_test.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package llm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNativeConversationReplayWithRepairContract(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": "version: 3\n",
|
||||||
|
"prompts.yml": `profiles:
|
||||||
|
- id: proposal-profile
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: proposal-model
|
||||||
|
- id: repair-profile
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
model: repair-model
|
||||||
|
prompts:
|
||||||
|
- id: proposal-fixture
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: 'System {{.Subject}}'
|
||||||
|
- role: user
|
||||||
|
content: 'Proposal {{.Subject}}'
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: correction_set.v1.json
|
||||||
|
- id: repair-fixture
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Repair {{.Finding}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`,
|
||||||
|
}
|
||||||
|
for name, content := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var requests []struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct{ Role, Content string } `json:"messages"`
|
||||||
|
ResponseFormat map[string]any `json:"response_format"`
|
||||||
|
}
|
||||||
|
outputs := []string{`{"corrections":[]}`, `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.8}}`, `{"proposal_ref":1,"action":"withdraw","correction":null}`}
|
||||||
|
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct{ Role, Content string } `json:"messages"`
|
||||||
|
ResponseFormat map[string]any `json:"response_format"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
requests = append(requests, request)
|
||||||
|
content, _ := json.Marshal(outputs[len(requests)-1])
|
||||||
|
body := fmt.Sprintf(`{"model":%q,"choices":[{"message":{"content":%s}}]}`, request.Model, content)
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||||
|
})
|
||||||
|
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
base := pk.RunRequest{PromptID: "proposal-fixture", ProfileID: "proposal-profile", Vars: map[string]string{"Subject": "source"}}
|
||||||
|
original, err := runtime.Engine.PrepareExecution(ctx, base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prefix := original.Details().Messages
|
||||||
|
initial, err := runtime.Engine.RunPrepared(ctx, original)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
suffix := []pk.RenderedMessage{{Role: pk.RoleAssistant, Content: initial.RawOutput}}
|
||||||
|
contract, err := OutputContract(responseschema.CorrectionRepairKey, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, finding := range []string{"first", "second"} {
|
||||||
|
instruction, err := runtime.Engine.Prepare(ctx, pk.RunRequest{PromptID: "repair-fixture", ProfileID: "repair-profile", Vars: map[string]string{"Finding": finding}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
suffix = append(suffix, instruction.Messages...)
|
||||||
|
req := base
|
||||||
|
req.ProfileID = "repair-profile"
|
||||||
|
req.AppendedMessages = suffix
|
||||||
|
req.Validation = contract
|
||||||
|
handle, err := runtime.Engine.PrepareExecution(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
details := handle.Details()
|
||||||
|
if !reflect.DeepEqual(details.Messages[:len(prefix)], prefix) || !reflect.DeepEqual(details.Messages[len(prefix):], suffix) {
|
||||||
|
t.Fatalf("prepared conversation = %#v", details.Messages)
|
||||||
|
}
|
||||||
|
result, err := runtime.Engine.RunPrepared(ctx, handle)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !result.Validation.IsValid {
|
||||||
|
t.Fatalf("invalid result: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
suffix = append(suffix, pk.RenderedMessage{Role: pk.RoleAssistant, Content: result.RawOutput})
|
||||||
|
}
|
||||||
|
if len(requests) != 3 || requests[0].Model != "proposal-model" || requests[1].Model != "repair-model" || requests[2].Model != "repair-model" {
|
||||||
|
t.Fatalf("models: %+v", requests)
|
||||||
|
}
|
||||||
|
for i, wantLen := range []int{2, 4, 6} {
|
||||||
|
req := requests[i]
|
||||||
|
if len(req.Messages) != wantLen {
|
||||||
|
t.Fatalf("request %d messages: %+v", i, req.Messages)
|
||||||
|
}
|
||||||
|
for j, message := range req.Messages {
|
||||||
|
if message.Role != string(append(prefix, suffix...)[j].Role) || message.Content != append(prefix, suffix...)[j].Content {
|
||||||
|
t.Fatalf("request %d message %d: %+v", i, j, message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(req.ResponseFormat)
|
||||||
|
expected := "corrections"
|
||||||
|
if i > 0 {
|
||||||
|
expected = "proposal_ref"
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(encoded), expected) {
|
||||||
|
t.Fatalf("request %d contract: %s", i, encoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
31
internal/framework/proposal_generation/repair.go
Normal file
31
internal/framework/proposal_generation/repair.go
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
package proposal_generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RepairResponse is one model response for an original proposal reference.
|
||||||
|
type RepairResponse struct {
|
||||||
|
ProposalRef int `json:"proposal_ref"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
Correction *StructuredCorrectionProposal `json:"correction"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeRepairResponse checks correspondence after structural schema validation.
|
||||||
|
func DecodeRepairResponse(content []byte, expectedRef int) (RepairResponse, error) {
|
||||||
|
var response RepairResponse
|
||||||
|
if err := json.Unmarshal(content, &response); err != nil {
|
||||||
|
return RepairResponse{}, invalidRepairResponse(err)
|
||||||
|
}
|
||||||
|
if response.ProposalRef != expectedRef || (response.Action != "revise" || response.Correction == nil) && (response.Action != "withdraw" || response.Correction != nil) {
|
||||||
|
return RepairResponse{}, invalidRepairResponse(nil)
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func invalidRepairResponse(cause error) error {
|
||||||
|
return &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutput, Message: "correction repair response is unusable", Cause: cause}
|
||||||
|
}
|
||||||
35
internal/framework/proposal_generation/repair_test.go
Normal file
35
internal/framework/proposal_generation/repair_test.go
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
package proposal_generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecodeRepairResponse(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, content string
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{"revision with deletion", `{"proposal_ref":2,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.5}}`, true},
|
||||||
|
{"withdrawal", `{"proposal_ref":2,"action":"withdraw","correction":null}`, true},
|
||||||
|
{"wrong reference", `{"proposal_ref":1,"action":"withdraw","correction":null}`, false},
|
||||||
|
{"missing revision", `{"proposal_ref":2,"action":"revise","correction":null}`, false},
|
||||||
|
{"withdrawal with revision", `{"proposal_ref":2,"action":"withdraw","correction":{"id":1}}`, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := DecodeRepairResponse([]byte(tc.content), 2)
|
||||||
|
if tc.valid && err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !tc.valid {
|
||||||
|
var failure *structuredoutput.Failure
|
||||||
|
if !errors.As(err, &failure) || failure.Reason != outcomes.ReasonInvalidOutput || failure.Class != structuredoutput.FailureRecoverable {
|
||||||
|
t.Fatalf("failure = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"proposal_ref",
|
||||||
|
"action",
|
||||||
|
"correction"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"proposal_ref": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
},
|
||||||
|
"action": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"revise",
|
||||||
|
"withdraw"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"correction": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": false,
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"original_text",
|
||||||
|
"corrected_text",
|
||||||
|
"confidence"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 1
|
||||||
|
},
|
||||||
|
"original_text": {
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
},
|
||||||
|
"corrected_text": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"confidence": {
|
||||||
|
"type": "number",
|
||||||
|
"minimum": 0,
|
||||||
|
"maximum": 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,9 +16,11 @@ type Key string
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
CorrectionSetKey Key = "correction_set"
|
CorrectionSetKey Key = "correction_set"
|
||||||
|
CorrectionRepairKey Key = "correction_repair"
|
||||||
ValidatorDecisionSetKey Key = "validator_decision_set"
|
ValidatorDecisionSetKey Key = "validator_decision_set"
|
||||||
AssetRoot = "assets"
|
AssetRoot = "assets"
|
||||||
correctionSetSchemaID = "audita.correction_set"
|
correctionSetSchemaID = "audita.correction_set"
|
||||||
|
correctionRepairSchemaID = "audita.correction_repair"
|
||||||
validatorDecisionSchemaID = "audita.validator_decision_set"
|
validatorDecisionSchemaID = "audita.validator_decision_set"
|
||||||
schemaVersionV1 = "v1"
|
schemaVersionV1 = "v1"
|
||||||
)
|
)
|
||||||
@@ -33,6 +35,8 @@ func AssetPath(key Key) (string, bool) {
|
|||||||
switch key {
|
switch key {
|
||||||
case CorrectionSetKey:
|
case CorrectionSetKey:
|
||||||
return "correction_set.v1.json", true
|
return "correction_set.v1.json", true
|
||||||
|
case CorrectionRepairKey:
|
||||||
|
return "correction_repair.v1.json", true
|
||||||
case ValidatorDecisionSetKey:
|
case ValidatorDecisionSetKey:
|
||||||
return "validator_decision_set.v1.json", true
|
return "validator_decision_set.v1.json", true
|
||||||
default:
|
default:
|
||||||
@@ -59,6 +63,7 @@ func (s Schema) DiagnosticsMap() map[string]any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var registry = map[Key]Schema{
|
var registry = map[Key]Schema{
|
||||||
|
CorrectionRepairKey: mustBuildSchemaFromAsset(correctionRepairSchemaID, schemaVersionV1, "audita_correction_repair_v1", "assets/correction_repair.v1.json"),
|
||||||
CorrectionSetKey: mustBuildSchemaFromAsset(
|
CorrectionSetKey: mustBuildSchemaFromAsset(
|
||||||
correctionSetSchemaID,
|
correctionSetSchemaID,
|
||||||
schemaVersionV1,
|
schemaVersionV1,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestPromptKitSchemaPathsUseCanonicalAssets(t *testing.T) {
|
func TestPromptKitSchemaPathsUseCanonicalAssets(t *testing.T) {
|
||||||
for _, key := range []Key{CorrectionSetKey, ValidatorDecisionSetKey} {
|
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
|
||||||
path, ok := AssetPath(key)
|
path, ok := AssetPath(key)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("missing path for %q", key)
|
t.Fatalf("missing path for %q", key)
|
||||||
@@ -81,7 +81,7 @@ func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
||||||
for _, key := range []Key{CorrectionSetKey, ValidatorDecisionSetKey} {
|
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
|
||||||
schema, ok := Lookup(key)
|
schema, ok := Lookup(key)
|
||||||
if !ok {
|
if !ok {
|
||||||
t.Fatalf("missing schema %q", key)
|
t.Fatalf("missing schema %q", key)
|
||||||
|
|||||||
Reference in New Issue
Block a user