Separate output contract inputs from resolved values
This commit is contained in:
11
convert.go
11
convert.go
@@ -318,21 +318,16 @@ func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func toDomainOutputContractPtr(contract *OutputContract) *domain.OutputContract {
|
func toDomainOutputContractPtr(contract *OutputContractInput) *domain.OutputContractInput {
|
||||||
if contract == nil {
|
if contract == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
out := toDomainOutputContract(*contract)
|
return &domain.OutputContractInput{
|
||||||
return &out
|
|
||||||
}
|
|
||||||
|
|
||||||
func toDomainOutputContract(contract OutputContract) domain.OutputContract {
|
|
||||||
return domain.OutputContract{
|
|
||||||
Format: domain.OutputFormat(contract.Format),
|
Format: domain.OutputFormat(contract.Format),
|
||||||
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
ValidationMode: domain.ValidationMode(contract.ValidationMode),
|
||||||
Schema: contract.Schema,
|
Schema: contract.Schema,
|
||||||
SchemaPath: contract.SchemaPath,
|
SchemaPath: contract.SchemaPath,
|
||||||
RepairAttempts: contract.RepairAttempts,
|
RepairAttempts: copyIntPtr(contract.RepairAttempts),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -261,6 +261,9 @@ with no change to actual-attempt counters or the repair loop.
|
|||||||
|
|
||||||
## Stage 3 — Separate Runtime Contract Inputs From Resolved Contracts
|
## Stage 3 — Separate Runtime Contract Inputs From Resolved Contracts
|
||||||
|
|
||||||
|
Status: complete. Internal use-case tests and the facade build passed; existing
|
||||||
|
public test and example call sites remain scheduled for migration below.
|
||||||
|
|
||||||
Focus: `types.go`, `convert.go`, `internal/domain/domain.go`, request ownership,
|
Focus: `types.go`, `convert.go`, `internal/domain/domain.go`, request ownership,
|
||||||
`internal/usecase/runner.go`, and focused runtime tests.
|
`internal/usecase/runner.go`, and focused runtime tests.
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ type RunRequest struct {
|
|||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
Validation *OutputContract
|
Validation *OutputContractInput
|
||||||
AppendedMessages []RenderedMessage
|
AppendedMessages []RenderedMessage
|
||||||
TransientRetries *int
|
TransientRetries *int
|
||||||
Observer ExecutionObserver `json:"-"`
|
Observer ExecutionObserver `json:"-"`
|
||||||
@@ -261,6 +261,15 @@ type OutputContract struct {
|
|||||||
RepairAttempts int `yaml:"repair_attempts"`
|
RepairAttempts int `yaml:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OutputContractInput is a request replacement before repair defaults resolve.
|
||||||
|
type OutputContractInput struct {
|
||||||
|
Format OutputFormat
|
||||||
|
ValidationMode ValidationMode
|
||||||
|
Schema any
|
||||||
|
SchemaPath string
|
||||||
|
RepairAttempts *int
|
||||||
|
}
|
||||||
|
|
||||||
// ResourceBinding identifies a resource in its originating filesystem.
|
// ResourceBinding identifies a resource in its originating filesystem.
|
||||||
type ResourceBinding struct {
|
type ResourceBinding struct {
|
||||||
FS fs.FS
|
FS fs.FS
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"go.maximumdirect.net/promptkit/internal/domain"
|
"go.maximumdirect.net/promptkit/internal/domain"
|
||||||
|
"go.maximumdirect.net/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
type outputContractTestCollaborators struct {
|
type outputContractTestCollaborators struct {
|
||||||
@@ -38,7 +39,7 @@ func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) {
|
|||||||
|
|
||||||
func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
|
func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
|
||||||
value := map[string]any{"properties": map[string]any{"name": map[string]any{"type": "string"}}}
|
value := map[string]any{"properties": map[string]any{"name": map[string]any{"type": "string"}}}
|
||||||
override := domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value}
|
override := domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: value}
|
||||||
contract, err := resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override)
|
contract, err := resolveOutputContract(context.Background(), &domain.PromptDefinition{}, &override)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -59,26 +60,26 @@ func TestResolveInlineSchemaOwnsRuntimeValue(t *testing.T) {
|
|||||||
func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
|
func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
override domain.OutputContract
|
override domain.OutputContractInput
|
||||||
want domain.OutputContract
|
want domain.OutputContract
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "empty replacement format defaults to text",
|
name: "empty replacement format defaults to text",
|
||||||
override: domain.OutputContract{ValidationMode: domain.ValidationNone},
|
override: domain.OutputContractInput{ValidationMode: domain.ValidationNone},
|
||||||
want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
want: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "markdown basic replacement",
|
name: "markdown basic replacement",
|
||||||
override: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
override: domain.OutputContractInput{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
||||||
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic},
|
want: domain.OutputContract{Format: domain.FormatMarkdown, ValidationMode: domain.ValidationBasic, RepairAttempts: 1},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "json replacement preserves non-schema fields",
|
name: "json replacement preserves non-schema fields",
|
||||||
override: domain.OutputContract{
|
override: domain.OutputContractInput{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
ValidationMode: domain.ValidationJSON,
|
ValidationMode: domain.ValidationJSON,
|
||||||
SchemaPath: "ignored.json",
|
SchemaPath: "ignored.json",
|
||||||
RepairAttempts: 2,
|
RepairAttempts: repairBudget(2),
|
||||||
},
|
},
|
||||||
want: domain.OutputContract{
|
want: domain.OutputContract{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
@@ -121,14 +122,14 @@ func TestRunnerPreparationNormalizesOutputContractConsistently(t *testing.T) {
|
|||||||
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) {
|
func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
override domain.OutputContract
|
override domain.OutputContractInput
|
||||||
}{
|
}{
|
||||||
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
|
{name: "unsupported format", override: domain.OutputContractInput{Format: "binary", ValidationMode: domain.ValidationNone}},
|
||||||
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
|
{name: "empty validation mode", override: domain.OutputContractInput{Format: domain.FormatText}},
|
||||||
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
|
{name: "negative repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(-1)}},
|
||||||
{name: "repair attempts above maximum", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 4}},
|
{name: "repair attempts above maximum", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(4)}},
|
||||||
{name: "none validation with repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: 1}},
|
{name: "none validation with repair attempts", override: domain.OutputContractInput{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: repairBudget(1)}},
|
||||||
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
{name: "json schema without path", override: domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -175,7 +176,7 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
Inputs: singleInputRef(),
|
Inputs: singleInputRef(),
|
||||||
Validation: &domain.OutputContract{
|
Validation: &domain.OutputContractInput{
|
||||||
Format: domain.OutputFormat("binary"),
|
Format: domain.OutputFormat("binary"),
|
||||||
ValidationMode: domain.ValidationNone,
|
ValidationMode: domain.ValidationNone,
|
||||||
},
|
},
|
||||||
@@ -189,6 +190,60 @@ func TestRunnerRunRejectsInvalidOutputContractBeforeAdmission(t *testing.T) {
|
|||||||
assertOutputContractCompletionSkipped(t, collaborators)
|
assertOutputContractCompletionSkipped(t, collaborators)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func repairBudget(value int) *int { return &value }
|
||||||
|
|
||||||
|
func TestReplacementRepairExecution(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
budget *int
|
||||||
|
wantCalls int
|
||||||
|
wantStatus domain.ValidationStatus
|
||||||
|
}{
|
||||||
|
{"default", nil, 2, domain.ValidationPassed},
|
||||||
|
{"explicit zero", repairBudget(0), 1, domain.ValidationFailed},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &sequenceLLM{responses: []*domain.GenerateResponse{{Content: "not json"}, {Content: `{"ok":true}`}}}
|
||||||
|
runner := NewRunnerWithRepairer(&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
defaultArtifactReader(), defaultRenderer(), client, validate.NewValidator(), NewDefaultOutputRepairer(client), nil)
|
||||||
|
input := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON, RepairAttempts: tc.budget}
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), Validation: input})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
input.ValidationMode = domain.ValidationNone
|
||||||
|
input.RepairAttempts = repairBudget(3)
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result.Validation.Status != tc.wantStatus || len(client.requests) != tc.wantCalls || result.Validation.RepairAttempts != tc.wantCalls-1 {
|
||||||
|
t.Fatalf("result = %+v, calls = %d, error = %v", result, len(client.requests), err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveReplacementRepairBudget(t *testing.T) {
|
||||||
|
configured := &domain.PromptDefinition{Validation: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 3}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
input domain.OutputContractInput
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"none", domain.OutputContractInput{ValidationMode: domain.ValidationNone}, 0},
|
||||||
|
{"basic", domain.OutputContractInput{ValidationMode: domain.ValidationBasic}, 1},
|
||||||
|
{"json", domain.OutputContractInput{ValidationMode: domain.ValidationJSON}, 1},
|
||||||
|
{"schema", domain.OutputContractInput{ValidationMode: domain.ValidationJSONSchema, Schema: true}, 1},
|
||||||
|
{"explicit zero", domain.OutputContractInput{ValidationMode: domain.ValidationBasic, RepairAttempts: repairBudget(0)}, 0},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, err := resolveOutputContract(context.Background(), configured, &tc.input)
|
||||||
|
if err != nil || got.RepairAttempts != tc.want {
|
||||||
|
t.Fatalf("contract = %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
func assertOutputContractCompletionSkipped(t *testing.T, collaborators outputContractTestCollaborators) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
if collaborators.artifacts.calls != 0 || collaborators.renderer.calls != 0 ||
|
||||||
|
|||||||
@@ -685,10 +685,10 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveOutputContract(ctx context.Context, def *domain.PromptDefinition, override *domain.OutputContract) (domain.OutputContract, error) {
|
func resolveOutputContract(ctx context.Context, def *domain.PromptDefinition, override *domain.OutputContractInput) (domain.OutputContract, error) {
|
||||||
contract := def.Validation
|
contract := def.Validation
|
||||||
if override != nil {
|
if override != nil {
|
||||||
contract = *override
|
contract = domain.OutputContract{Format: override.Format, ValidationMode: override.ValidationMode, Schema: override.Schema, SchemaPath: override.SchemaPath, RepairAttempts: domain.ResolveRepairAttempts(override.ValidationMode, override.RepairAttempts)}
|
||||||
if contract.Format == "" {
|
if contract.Format == "" {
|
||||||
contract.Format = domain.FormatText
|
contract.Format = domain.FormatText
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
|
|||||||
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
|
if err != nil || configured.File == nil || configured.File.Path != "origin/configured.json" {
|
||||||
t.Fatalf("configured binding = %+v, %v", configured, err)
|
t.Fatalf("configured binding = %+v, %v", configured, err)
|
||||||
}
|
}
|
||||||
inline := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
|
inline := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, Schema: false}
|
||||||
contract, err := resolveOutputContract(context.Background(), def, inline)
|
contract, err := resolveOutputContract(context.Background(), def, inline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -30,7 +30,7 @@ func TestValidationBindingUsesEffectiveContractAndPromptRoot(t *testing.T) {
|
|||||||
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
|
if err != nil || selected.File != nil || selected.Contract.Schema != false || selected.Contract.SchemaFile != nil {
|
||||||
t.Fatalf("inline replacement = %+v, %v", selected, err)
|
t.Fatalf("inline replacement = %+v, %v", selected, err)
|
||||||
}
|
}
|
||||||
path := &domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
|
path := &domain.OutputContractInput{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema, SchemaPath: "runtime.json"}
|
||||||
contract, err = resolveOutputContract(context.Background(), def, path)
|
contract, err = resolveOutputContract(context.Background(), def, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
16
types.go
16
types.go
@@ -160,7 +160,7 @@ type RunRequest struct {
|
|||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
// Validation optionally replaces the prompt's complete output contract. It
|
// Validation optionally replaces the prompt's complete output contract. It
|
||||||
// does not merge individual fields. Nil uses the prompt contract.
|
// does not merge individual fields. Nil uses the prompt contract.
|
||||||
Validation *OutputContract
|
Validation *OutputContractInput
|
||||||
// AppendedMessages are already-rendered messages appended in caller order
|
// AppendedMessages are already-rendered messages appended in caller order
|
||||||
// after every prompt definition message. Promptkit neither templates nor
|
// after every prompt definition message. Promptkit neither templates nor
|
||||||
// resolves files in them. Content must be valid UTF-8 and is preserved
|
// resolves files in them. Content must be valid UTF-8 and is preserved
|
||||||
@@ -490,8 +490,7 @@ type ExecutionTargetPresence struct {
|
|||||||
TimeoutSeconds bool `json:"timeout_seconds"`
|
TimeoutSeconds bool `json:"timeout_seconds"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// OutputContract defines output and validation requirements and has a stable
|
// OutputContract describes resolved output and validation requirements.
|
||||||
// JSON representation.
|
|
||||||
//
|
//
|
||||||
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
||||||
// does not merge fields. The public Engine performs bounded correction after a
|
// does not merge fields. The public Engine performs bounded correction after a
|
||||||
@@ -520,6 +519,17 @@ type OutputContract struct {
|
|||||||
RepairAttempts int `json:"repair_attempts"`
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OutputContractInput replaces a prompt's complete output contract. An omitted
|
||||||
|
// repair budget defaults to one for active validation and zero for none;
|
||||||
|
// an explicit zero disables repair. Schema selection follows OutputContract.
|
||||||
|
type OutputContractInput struct {
|
||||||
|
Format OutputFormat `json:"format"`
|
||||||
|
ValidationMode ValidationMode `json:"validation_mode"`
|
||||||
|
Schema any `json:"schema,omitempty"`
|
||||||
|
SchemaPath string `json:"schema_path"`
|
||||||
|
RepairAttempts *int `json:"repair_attempts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// ValidationResult represents a completed output check and has a stable JSON
|
// ValidationResult represents a completed output check and has a stable JSON
|
||||||
// representation. An operational inability to perform validation is returned
|
// representation. An operational inability to perform validation is returned
|
||||||
// as ErrValidation instead of a ValidationResult.
|
// as ErrValidation instead of a ValidationResult.
|
||||||
|
|||||||
Reference in New Issue
Block a user