Resolve repair budgets from validation modes
This commit is contained in:
@@ -3,6 +3,7 @@ package configuration
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -117,11 +118,96 @@ func TestInspectPromptResourcePresence(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||
t.Fatalf("prompt resource = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfiguredRepairBudgets(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, mode string
|
||||
budget any
|
||||
include bool
|
||||
want int
|
||||
invalid bool
|
||||
}{
|
||||
{name: "none omitted", mode: "none", want: 0},
|
||||
{name: "basic omitted", mode: "basic", want: 1},
|
||||
{name: "json omitted", mode: "json", want: 1},
|
||||
{name: "schema omitted", mode: "json_schema", want: 1},
|
||||
{name: "null", mode: "json", include: true, budget: nil, want: 1},
|
||||
{name: "zero", mode: "basic", include: true, budget: 0, want: 0},
|
||||
{name: "two", mode: "basic", include: true, budget: 2, want: 2},
|
||||
{name: "three", mode: "basic", include: true, budget: 3, want: 3},
|
||||
{name: "negative", mode: "basic", include: true, budget: -1, invalid: true},
|
||||
{name: "excess", mode: "basic", include: true, budget: 4, invalid: true},
|
||||
{name: "none positive", mode: "none", include: true, budget: 1, invalid: true},
|
||||
{name: "unknown mode", mode: "unknown", invalid: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
output := map[string]any{"format": "text", "validation_mode": tc.mode}
|
||||
if tc.mode == "json_schema" {
|
||||
output["schema"] = true
|
||||
}
|
||||
if tc.include {
|
||||
output["repair_attempts"] = tc.budget
|
||||
}
|
||||
state, err := load(mapping("repair", map[string]any{"prompts": []any{map[string]any{
|
||||
"id": "test", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output,
|
||||
}}}), nil)
|
||||
if tc.invalid {
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid output contract")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("test")
|
||||
if err != nil || definition.Validation.RepairAttempts != tc.want {
|
||||
t.Fatalf("captured budget: %+v, %v", definition, err)
|
||||
}
|
||||
report, err := state.InspectPrompt(context.Background(), "test")
|
||||
if err != nil || report.Output.RepairAttempts != tc.want {
|
||||
t.Fatalf("inspection: %+v, %v", report, err)
|
||||
}
|
||||
if got := state.result.Config.Prompts[0].Output.RepairAttempts; got == nil || *got != tc.want {
|
||||
t.Fatalf("effective budget = %v", got)
|
||||
}
|
||||
var effective strings.Builder
|
||||
if err := state.result.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), fmt.Sprintf("repair_attempts: %d", tc.want)) {
|
||||
t.Fatalf("effective inspection = %q, %v", effective.String(), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNullClearsInheritedRepairBudget(t *testing.T) {
|
||||
state, err := load(mapping("inherited repair", map[string]any{"prompts": []any{
|
||||
map[string]any{"id": "parent", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 3}},
|
||||
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "output": map[string]any{"repair_attempts": nil}},
|
||||
}}), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
child, _, err := state.prompt("child")
|
||||
if err != nil || child.Validation.RepairAttempts != 1 {
|
||||
t.Fatalf("child = %+v, %v", child, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYAMLRepairBudgetNull(t *testing.T) {
|
||||
state, err := load(configkit.Input{Source: configkit.Bytes("prompt", []byte("prompts:\n - id: example\n messages:\n - role: user\n content: hello\n output:\n format: text\n validation_mode: basic\n repair_attempts: null\n"))}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
definition, _, err := state.prompt("example")
|
||||
if err != nil || definition.Validation.RepairAttempts != 1 {
|
||||
t.Fatalf("prompt = %+v, %v", definition, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
||||
files := fstest.MapFS{
|
||||
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
||||
|
||||
Reference in New Issue
Block a user