Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
@@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := testValidationClient(t, transport)
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The transport supplies provider responses; PromptKit performs structured
|
||||
// validation and Audita decodes the same response through the role client.
|
||||
func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, response string
|
||||
indices []int
|
||||
want []DecisionDisposition
|
||||
reasons []string
|
||||
failure outcomes.ReasonCode
|
||||
}{
|
||||
{
|
||||
name: "usable decision beside extra",
|
||||
response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7},
|
||||
want: []DecisionDisposition{DispositionApproved},
|
||||
reasons: []string{ReasonApproved},
|
||||
},
|
||||
{
|
||||
name: "usable siblings beside duplicate and omission",
|
||||
response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7, 11, 14, 23},
|
||||
want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected},
|
||||
reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"},
|
||||
failure: outcomes.ReasonValidationDecisionsInvalid,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
content, _ := json.Marshal(tc.response)
|
||||
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
||||
}))
|
||||
var candidates []proposals.EnrichedCorrectionProposal
|
||||
for _, index := range tc.indices {
|
||||
candidates = append(candidates, mk(index, "gestures", "Jesters"))
|
||||
}
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &captureValidationDiagnosticsWriter{}
|
||||
req := makeReq(candidates)
|
||||
req.LLMClient = client
|
||||
req.Interactions = sink
|
||||
result, err := validator.Validate(context.Background(), req)
|
||||
if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d error=%v", result, calls, err)
|
||||
}
|
||||
for i, decision := range result.Decisions {
|
||||
if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] {
|
||||
t.Fatalf("decision %d: %+v", i, decision)
|
||||
}
|
||||
}
|
||||
operation := result.Operations[0]
|
||||
if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) {
|
||||
t.Fatalf("operation: %+v", operation)
|
||||
}
|
||||
payload := sink.lastResponsePayload.(map[string]any)
|
||||
diagnostics := payload["reconciliation"].(validationReconciliation)
|
||||
if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response {
|
||||
t.Fatalf("reference evidence changed: %+v", payload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user