Separate output contract inputs from resolved values

This commit is contained in:
2026-11-21 14:12:34 +00:00
parent dad302f1ff
commit 38a2dcca52
7 changed files with 103 additions and 31 deletions

View File

@@ -6,6 +6,7 @@ import (
"testing"
"go.maximumdirect.net/promptkit/internal/domain"
"go.maximumdirect.net/promptkit/internal/validate"
)
type outputContractTestCollaborators struct {
@@ -38,7 +39,7 @@ func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) {
func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
value := map[string]any{"properties": map[string]any{"name": map[string]any{"type": "string"}}}
override := domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value}
override := domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value}
contract, err := resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override)
if err != nil {
t.Fatal(err)
@@ -59,26 +60,26 @@ func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
tests := []struct {
name string
override domain.OutputContract
override domain.OutputContractInput
want domain.OutputContract
}{
{
name: "empty replacement format defaults to text",
override: domain.OutputContract{ValidationMode: domain.ValidationNone},
override: domain.OutputContractInput{ValidationMode: domain.ValidationNone},
want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
},
{
name: "markdown basic replacement",
override: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
override: domain.OutputContractInput{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic, RepairAttempts: 1},
},
{
name: "json replacement preserves non-schema fields",
override: domain.OutputContract{
override: domain.OutputContractInput{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSON,
SchemaPath: "ignored.json",
RepairAttempts: 2,
RepairAttempts: repairBudget(2),
},
want: domain.OutputContract{
Format: domain.FormatJSON,
@@ -121,14 +122,14 @@ func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) {
tests := []struct {
name string
override domain.OutputContract
override domain.OutputContractInput
}{
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
{name: "repair attempts above maximum", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 4}},
{name: "none validation with repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: 1}},
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
{name: "unsupported format", override: domain.OutputContractInput{Format: "binary", ValidationMode: domain.ValidationNone}},
{name: "empty validation mode", override: domain.OutputContractInput{Format: domain.FormatText}},
{name: "negative repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(-1)}},
{name: "repair attempts above maximum", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(4)}},
{name: "none validation with repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(1)}},
{name: "json schema without path", override: domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
}
for _, tt := range tests {
@@ -175,7 +176,7 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Validation: &domain.OutputContract{
Validation: &domain.OutputContractInput{
Format: domain.OutputFormat("binary"),
ValidationMode: domain.ValidationNone,
},
@@ -189,6 +190,60 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
assertOutputContractCompletionSkipped(t, collaborators)
}
func repairBudget(value int) *int { return &value }
func TestReplacementRepairExecution(t *testing.T) {
for _, tc := range []struct {
name string
budget *int
wantCalls int
wantStatus domain.ValidationStatus
}{
{"default", nil, 2, domain.ValidationPassed},
{"explicit zero", repairBudget(0), 1, domain.ValidationFailed},
} {
t.Run(tc.name, func(t *testing.T) {
client := &sequenceLLM{responses: []*domain.GenerateResponse{{Content: "not json"}, {Content: `{"ok":true}`}}}
runner := NewRunnerWithRepairer(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(), defaultRenderer(), client, validate.NewValidator(), NewDefaultOutputRepairer(client), nil)
input := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, RepairAttempts: tc.budget}
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Validation: input})
if err != nil {
t.Fatal(err)
}
input.ValidationMode = domain.ValidationNone
input.RepairAttempts = repairBudget(3)
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil || result.Validation.Status != tc.wantStatus || len(client.requests) != tc.wantCalls || result.Validation.RepairAttempts != tc.wantCalls-1 {
t.Fatalf("result = %+v, calls = %d, error = %v", result, len(client.requests), err)
}
})
}
}
func TestResolveReplacementRepairBudget(t *testing.T) {
configured := &domain.PromptDefinition{Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 3}}
for _, tc := range []struct {
name string
input domain.OutputContractInput
want int
}{
{"none", domain.OutputContractInput{ValidationMode: domain.ValidationNone}, 0},
{"basic", domain.OutputContractInput{ValidationMode: domain.ValidationBasic}, 1},
{"json", domain.OutputContractInput{ValidationMode: domain.ValidationJSON}, 1},
{"schema", domain.OutputContractInput{ValidationMode: domain.ValidationJSONSchema, Schema: true}, 1},
{"explicit zero", domain.OutputContractInput{ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(0)}, 0},
} {
t.Run(tc.name, func(t *testing.T) {
got, err := resolveOutputContract(context.Background(), configured, &tc.input)
if err != nil || got.RepairAttempts != tc.want {
t.Fatalf("contract = %+v, %v", got, err)
}
})
}
}
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
t.Helper()
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||