Resolve repair budgets from validation modes
This commit is contained in:
@@ -283,7 +283,7 @@ func fromBackendResource(v *configuration.BackendResource) *BackendInspection {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
func fromPromptResource(v *configuration.PromptResource) *PromptInspection {
|
||||||
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: copyIntPtr(v.Output.RepairAttempts)}}
|
out := &PromptInspection{PromptID: v.ID, PromptHash: v.Hash, DefaultProfileID: copyStringPtr(v.DefaultProfile), Description: copyStringPtr(v.Description), SessionID: copyStringPtr(v.SessionID), OutputContract: PromptOutputInspection{Format: OutputFormat(v.Output.Format), ValidationMode: ValidationMode(v.Output.ValidationMode), Schema: jsonvalue.CloneTrusted(v.Output.Schema), SchemaFile: copyStringPtr(v.Output.SchemaFile), SchemaPath: copyStringPtr(v.Output.SchemaPath), RepairAttempts: v.Output.RepairAttempts}}
|
||||||
if v.Inputs != nil {
|
if v.Inputs != nil {
|
||||||
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
inputs := make([]PromptInputInspection, len(*v.Inputs))
|
||||||
for i, input := range *v.Inputs {
|
for i, input := range *v.Inputs {
|
||||||
|
|||||||
@@ -233,6 +233,8 @@ validation, not an approval gate before Stage 2.
|
|||||||
|
|
||||||
## Stage 2 — Resolve Configuration Repair Defaults
|
## Stage 2 — Resolve Configuration Repair Defaults
|
||||||
|
|
||||||
|
Status: complete. Focused configuration, domain, and public source tests passed.
|
||||||
|
|
||||||
Focus: `internal/domain`, configuration output decoding/resolution/capture,
|
Focus: `internal/domain`, configuration output decoding/resolution/capture,
|
||||||
resource inspection, and their narrow tests.
|
resource inspection, and their narrow tests.
|
||||||
|
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ type promptOutputConfig struct {
|
|||||||
Schema any `yaml:"schema"`
|
Schema any `yaml:"schema"`
|
||||||
SchemaFile *string `yaml:"schema_file"`
|
SchemaFile *string `yaml:"schema_file"`
|
||||||
SchemaPath *string `yaml:"schema_path"`
|
SchemaPath *string `yaml:"schema_path"`
|
||||||
RepairAttempts int `yaml:"repair_attempts"`
|
RepairAttempts *int `yaml:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// fieldState distinguishes omission, an explicit value, and an explicit null.
|
// fieldState distinguishes omission, an explicit value, and an explicit null.
|
||||||
@@ -124,7 +124,6 @@ const (
|
|||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
promptRepair map[string]bool
|
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
backendRecords map[string]backendConfig
|
backendRecords map[string]backendConfig
|
||||||
@@ -239,7 +238,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
||||||
}
|
}
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
@@ -258,7 +257,6 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
promptRepair: make(map[string]bool),
|
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
@@ -328,7 +326,7 @@ type promptInputPresence struct {
|
|||||||
description fieldState
|
description fieldState
|
||||||
}
|
}
|
||||||
|
|
||||||
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) {
|
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
||||||
for promptIndex, prompt := range prompts {
|
for promptIndex, prompt := range prompts {
|
||||||
inputs := prompt.Inputs
|
inputs := prompt.Inputs
|
||||||
states := make([]promptInputPresence, len(inputs))
|
states := make([]promptInputPresence, len(inputs))
|
||||||
@@ -342,7 +340,6 @@ func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, name
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[indexedResourceID(namespace, prompt.ID)] = states
|
destination[indexedResourceID(namespace, prompt.ID)] = states
|
||||||
repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -247,7 +247,7 @@ func capturePromptOutput(output *promptOutputConfig, base configkit.Path) (domai
|
|||||||
contract := domain.OutputContract{
|
contract := domain.OutputContract{
|
||||||
Format: domain.OutputFormat(*output.Format),
|
Format: domain.OutputFormat(*output.Format),
|
||||||
ValidationMode: domain.ValidationMode(*output.ValidationMode),
|
ValidationMode: domain.ValidationMode(*output.ValidationMode),
|
||||||
RepairAttempts: output.RepairAttempts,
|
RepairAttempts: domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), output.RepairAttempts),
|
||||||
Schema: jsonvalue.CloneTrusted(output.Schema),
|
Schema: jsonvalue.CloneTrusted(output.Schema),
|
||||||
SchemaFile: stringPointerValue(output.SchemaFile),
|
SchemaFile: stringPointerValue(output.SchemaFile),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package configuration
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -117,11 +118,96 @@ func TestInspectPromptResourcePresence(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||||
t.Fatalf("prompt resource = %+v", got)
|
t.Fatalf("prompt resource = %+v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestConfiguredRepairBudgets(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, mode string
|
||||||
|
budget any
|
||||||
|
include bool
|
||||||
|
want int
|
||||||
|
invalid bool
|
||||||
|
}{
|
||||||
|
{name: "none omitted", mode: "none", want: 0},
|
||||||
|
{name: "basic omitted", mode: "basic", want: 1},
|
||||||
|
{name: "json omitted", mode: "json", want: 1},
|
||||||
|
{name: "schema omitted", mode: "json_schema", want: 1},
|
||||||
|
{name: "null", mode: "json", include: true, budget: nil, want: 1},
|
||||||
|
{name: "zero", mode: "basic", include: true, budget: 0, want: 0},
|
||||||
|
{name: "two", mode: "basic", include: true, budget: 2, want: 2},
|
||||||
|
{name: "three", mode: "basic", include: true, budget: 3, want: 3},
|
||||||
|
{name: "negative", mode: "basic", include: true, budget: -1, invalid: true},
|
||||||
|
{name: "excess", mode: "basic", include: true, budget: 4, invalid: true},
|
||||||
|
{name: "none positive", mode: "none", include: true, budget: 1, invalid: true},
|
||||||
|
{name: "unknown mode", mode: "unknown", invalid: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
output := map[string]any{"format": "text", "validation_mode": tc.mode}
|
||||||
|
if tc.mode == "json_schema" {
|
||||||
|
output["schema"] = true
|
||||||
|
}
|
||||||
|
if tc.include {
|
||||||
|
output["repair_attempts"] = tc.budget
|
||||||
|
}
|
||||||
|
state, err := load(mapping("repair", map[string]any{"prompts": []any{map[string]any{
|
||||||
|
"id": "test", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": output,
|
||||||
|
}}}), nil)
|
||||||
|
if tc.invalid {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid output contract")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition, _, err := state.prompt("test")
|
||||||
|
if err != nil || definition.Validation.RepairAttempts != tc.want {
|
||||||
|
t.Fatalf("captured budget: %+v, %v", definition, err)
|
||||||
|
}
|
||||||
|
report, err := state.InspectPrompt(context.Background(), "test")
|
||||||
|
if err != nil || report.Output.RepairAttempts != tc.want {
|
||||||
|
t.Fatalf("inspection: %+v, %v", report, err)
|
||||||
|
}
|
||||||
|
if got := state.result.Config.Prompts[0].Output.RepairAttempts; got == nil || *got != tc.want {
|
||||||
|
t.Fatalf("effective budget = %v", got)
|
||||||
|
}
|
||||||
|
var effective strings.Builder
|
||||||
|
if err := state.result.WriteEffective(&effective); err != nil || !strings.Contains(effective.String(), fmt.Sprintf("repair_attempts: %d", tc.want)) {
|
||||||
|
t.Fatalf("effective inspection = %q, %v", effective.String(), err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNullClearsInheritedRepairBudget(t *testing.T) {
|
||||||
|
state, err := load(mapping("inherited repair", map[string]any{"prompts": []any{
|
||||||
|
map[string]any{"id": "parent", "messages": []any{map[string]any{"role": "user", "content": "hello"}}, "output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 3}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "parent"}, "output": map[string]any{"repair_attempts": nil}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
child, _, err := state.prompt("child")
|
||||||
|
if err != nil || child.Validation.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("child = %+v, %v", child, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestYAMLRepairBudgetNull(t *testing.T) {
|
||||||
|
state, err := load(configkit.Input{Source: configkit.Bytes("prompt", []byte("prompts:\n - id: example\n messages:\n - role: user\n content: hello\n output:\n format: text\n validation_mode: basic\n repair_attempts: null\n"))}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
definition, _, err := state.prompt("example")
|
||||||
|
if err != nil || definition.Validation.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("prompt = %+v, %v", definition, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
||||||
|
|||||||
@@ -40,6 +40,10 @@ func resolvePrompts(entries []promptConfig, base configkit.Path, resolution *con
|
|||||||
if entry.Output != nil && entry.Output.SchemaPath != nil {
|
if entry.Output != nil && entry.Output.SchemaPath != nil {
|
||||||
recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath))
|
recordStringNormalization(&entry.Output.SchemaPath, path.Key("output").Key("schema_path"), resolution, strings.TrimSpace(*entry.Output.SchemaPath))
|
||||||
}
|
}
|
||||||
|
if output := entry.Output; output != nil && output.ValidationMode != nil && output.RepairAttempts == nil {
|
||||||
|
attempts := domain.ResolveRepairAttempts(domain.ValidationMode(*output.ValidationMode), nil)
|
||||||
|
configkit.Normalize(resolution, path.Key("output").Key("repair_attempts"), &output.RepairAttempts, &attempts, "Resolve repair budget")
|
||||||
|
}
|
||||||
for j := range entry.Messages {
|
for j := range entry.Messages {
|
||||||
message := &entry.Messages[j]
|
message := &entry.Messages[j]
|
||||||
messagePath := path.Key("messages").Index(j)
|
messagePath := path.Key("messages").Index(j)
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ type PromptOutputResource struct {
|
|||||||
Schema any
|
Schema any
|
||||||
SchemaFile *string
|
SchemaFile *string
|
||||||
SchemaPath *string
|
SchemaPath *string
|
||||||
RepairAttempts *int
|
RepairAttempts int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
|
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
|
||||||
@@ -95,11 +95,7 @@ func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResou
|
|||||||
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
|
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, Schema: entry.definition.Validation.Schema, SchemaFile: raw.Output.SchemaFile, SchemaPath: raw.Output.SchemaPath}
|
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, Schema: entry.definition.Validation.Schema, SchemaFile: raw.Output.SchemaFile, SchemaPath: raw.Output.SchemaPath, RepairAttempts: entry.definition.Validation.RepairAttempts}
|
||||||
if state.promptRepair[id] {
|
|
||||||
value := raw.Output.RepairAttempts
|
|
||||||
result.Output.RepairAttempts = &value
|
|
||||||
}
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,19 @@ import (
|
|||||||
|
|
||||||
const maxOutputRepairAttempts = 3
|
const maxOutputRepairAttempts = 3
|
||||||
|
|
||||||
|
// ResolveRepairAttempts selects a concrete budget for a validation mode.
|
||||||
|
func ResolveRepairAttempts(mode ValidationMode, attempts *int) int {
|
||||||
|
if attempts != nil {
|
||||||
|
return *attempts
|
||||||
|
}
|
||||||
|
switch mode {
|
||||||
|
case ValidationBasic, ValidationJSON, ValidationJSONSchema:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ValidateOutputContract validates source-neutral output-contract invariants.
|
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||||
func ValidateOutputContract(contract OutputContract) error {
|
func ValidateOutputContract(contract OutputContract) error {
|
||||||
switch contract.Format {
|
switch contract.Format {
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ func TestTypedSourcesComposeAndCaptureValues(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID != "local" || prompt.Description == nil || *prompt.Description != "description" || prompt.Inputs == nil || len(*prompt.Inputs) != 1 || (*prompt.Inputs)[0].Name != "input" || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required || (*prompt.Inputs)[0].Description == nil || *(*prompt.Inputs)[0].Description != "" || len(prompt.Messages) != 2 || prompt.Messages[0].Content != "hello {{.name}}" || prompt.Messages[1].Content != "second" || prompt.Messages[0].CacheControl.TTL == nil || *prompt.Messages[0].CacheControl.TTL != "" || prompt.OutputContract.RepairAttempts == nil || *prompt.OutputContract.RepairAttempts != 0 {
|
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID != "local" || prompt.Description == nil || *prompt.Description != "description" || prompt.Inputs == nil || len(*prompt.Inputs) != 1 || (*prompt.Inputs)[0].Name != "input" || (*prompt.Inputs)[0].Required == nil || *(*prompt.Inputs)[0].Required || (*prompt.Inputs)[0].Description == nil || *(*prompt.Inputs)[0].Description != "" || len(prompt.Messages) != 2 || prompt.Messages[0].Content != "hello {{.name}}" || prompt.Messages[1].Content != "second" || prompt.Messages[0].CacheControl.TTL == nil || *prompt.Messages[0].CacheControl.TTL != "" || prompt.OutputContract.RepairAttempts != 0 {
|
||||||
t.Fatalf("prompt = %+v", prompt)
|
t.Fatalf("prompt = %+v", prompt)
|
||||||
}
|
}
|
||||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "child", Vars: map[string]string{"name": "reader"}})
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{PromptID: "child", Vars: map[string]string{"name": "reader"}})
|
||||||
|
|||||||
@@ -83,5 +83,5 @@ type PromptOutputInspection struct {
|
|||||||
Schema any `json:"schema,omitempty" yaml:"schema,omitempty"`
|
Schema any `json:"schema,omitempty" yaml:"schema,omitempty"`
|
||||||
SchemaFile *string `json:"schema_file,omitempty" yaml:"schema_file,omitempty"`
|
SchemaFile *string `json:"schema_file,omitempty" yaml:"schema_file,omitempty"`
|
||||||
SchemaPath *string `json:"schema_path,omitempty" yaml:"schema_path,omitempty"`
|
SchemaPath *string `json:"schema_path,omitempty" yaml:"schema_path,omitempty"`
|
||||||
RepairAttempts *int `json:"repair_attempts,omitempty" yaml:"repair_attempts,omitempty"`
|
RepairAttempts int `json:"repair_attempts" yaml:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user