Add module correction repair prompts
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestEngineCapturesAutomaticPromptSources(t *testing.T) {
|
||||
@@ -198,3 +199,31 @@ func TestEngineResolvesLinkedPromptMessageBesideDefinition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairPromptReplacementUsesMessageFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\nprofiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\n",
|
||||
"repair.txt": "Replacement {{.RepairRequestJSON}}",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := NewRuntime(context.Background(), loaded, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := runtime.Engine.Prepare(context.Background(), pk.RunRequest{PromptID: prompts.PromptIDModuleGrammarRepair, ProfileID: "fixture", Vars: map[string]string{"RepairRequestJSON": "fixture request"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "Replacement fixture request" || prepared.OutputContract.SchemaPath != "correction_repair.v1.json" {
|
||||
t.Fatalf("prepared replacement: %+v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
23
internal/framework/proposal_generation/repair_request.go
Normal file
23
internal/framework/proposal_generation/repair_request.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package proposal_generation
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// RepairFinding contains safe validator feedback for the current correction.
|
||||
type RepairFinding struct {
|
||||
Status string `json:"status,omitempty"`
|
||||
Validator string `json:"validator"`
|
||||
Reason string `json:"reason"`
|
||||
Explanation string `json:"explanation,omitempty"`
|
||||
}
|
||||
|
||||
// RepairRequest identifies the proposal version and its ordered findings.
|
||||
type RepairRequest struct {
|
||||
ProposalRef int `json:"proposal_ref"`
|
||||
Correction StructuredCorrectionProposal `json:"correction"`
|
||||
Findings []RepairFinding `json:"findings"`
|
||||
}
|
||||
|
||||
func (r RepairRequest) JSON() (string, error) {
|
||||
data, err := json.Marshal(r)
|
||||
return string(data), err
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRepairRequestJSONPreservesCurrentVersionAndFindings(t *testing.T) {
|
||||
request := RepairRequest{ProposalRef: 3, Correction: StructuredCorrectionProposal{SegmentReference: 2, OriginalText: " ", CorrectedText: "", Confidence: 0.7}, Findings: []RepairFinding{{Validator: "first", Reason: "rejected", Explanation: "detail"}, {Validator: "second", Status: "unavailable", Reason: "provider_failed"}}}
|
||||
text, err := request.JSON()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got RepairRequest
|
||||
if err := json.Unmarshal([]byte(text), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.ProposalRef != 3 || got.Correction != request.Correction || len(got.Findings) != 2 || got.Findings[0] != request.Findings[0] || got.Findings[1] != request.Findings[1] {
|
||||
t.Fatalf("request: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
jsonschema "github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
func TestPromptKitSchemaPathsUseCanonicalAssets(t *testing.T) {
|
||||
@@ -132,3 +134,41 @@ func TestDiagnosticsMapIncludesStableSchemaMetadataShapeForAllSchemas(t *testing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionRepairSchema(t *testing.T) {
|
||||
var document any
|
||||
if err := json.Unmarshal(MustLookup(CorrectionRepairKey).JSONSchema, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("repair.json", document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema, err := compiler.Compile("repair.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, content string
|
||||
valid bool
|
||||
}{
|
||||
{"delete whitespace", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0}}`, true},
|
||||
{"withdraw", `{"proposal_ref":1,"action":"withdraw","correction":null}`, true},
|
||||
{"missing correction", `{"proposal_ref":1,"action":"withdraw"}`, false},
|
||||
{"extra field", `{"proposal_ref":1,"action":"withdraw","correction":null,"extra":1}`, false},
|
||||
{"zero reference", `{"proposal_ref":0,"action":"withdraw","correction":null}`, false},
|
||||
{"unknown action", `{"proposal_ref":1,"action":"other","correction":null}`, false},
|
||||
{"empty original", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"","corrected_text":"x","confidence":1}}`, false},
|
||||
{"confidence outside range", `{"proposal_ref":1,"action":"revise","correction":{"id":1,"original_text":"x","corrected_text":"y","confidence":2}}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var value any
|
||||
if err := json.Unmarshal([]byte(tc.content), &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := schema.Validate(value); (err == nil) != tc.valid {
|
||||
t.Fatalf("validation = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user