base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Migrate validator tests to prepared plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-05 20:04:55 +00:00
parent 1fbd175e53
commit 9463c9463a
3 changed files with 80 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/usecase/runner_test.go
View File

@@ -240,14 +240,6 @@ type fakeValidator struct {
validateCalls int
}
func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
f.validateCalls++
if f.err != nil {
return domain.ValidationResult{}, f.err
}
return f.result, nil
}
func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) {
var schemaDocument any
if contract.ValidationMode == domain.ValidationJSONSchema {
@@ -261,17 +253,20 @@ func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.Out
schemaDocument = map[string]any{"type": "object"}
}
}
return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil
return &fakePreparedValidator{validator: f, schemaDocument: schemaDocument}, nil
}
type fakePreparedValidator struct {
validator *fakeValidator
contract domain.OutputContract
schemaDocument any
}
func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
return p.validator.Validate(ctx, artifact, p.contract)
p.validator.validateCalls++
if p.validator.err != nil {
return domain.ValidationResult{}, p.validator.err
}
return p.validator.result, nil
}
func (p *fakePreparedValidator) SchemaDocument() any {

121
internal/validate/standard_validator_test.go
View File

@@ -17,12 +17,21 @@ import (
"go.maximumdirect.net/promptkit/internal/domain"
)
func prepareForTest(t testing.TB, preparer ValidationPreparer, contract domain.OutputContract) PreparedValidation {
t.Helper()
plan, err := preparer.PrepareValidation(context.Background(), contract)
if err != nil {
t.Fatalf("prepare validation: %v", err)
}
return plan
}
func TestStandardValidatorNoneSkipped(t *testing.T) {
v := NewStandardValidator("")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationNone,
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -44,9 +53,9 @@ func TestStandardValidatorBasicSuccess(t *testing.T) {
{name: "invalid UTF-8 byte", body: []byte{0xff}},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
}).Validate(context.Background(), &domain.Artifact{Body: tc.body})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -68,9 +77,9 @@ func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
{name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
}).Validate(context.Background(), &domain.Artifact{Body: tc.body})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -110,7 +119,7 @@ func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)}, tc.contract)
res, err := prepareForTest(t, v, tc.contract).Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)})
if err != nil {
t.Fatalf("validate: %v", err)
}
@@ -124,9 +133,9 @@ func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) {
func TestStandardValidatorJSONSuccess(t *testing.T) {
v := NewStandardValidator("")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSON,
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -138,9 +147,9 @@ func TestStandardValidatorJSONSuccess(t *testing.T) {
func TestStandardValidatorJSONFailure(t *testing.T) {
v := NewStandardValidator("")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSON,
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -175,9 +184,9 @@ func TestJSONValidationChecksCompleteSyntaxWithoutChangingArtifact(t *testing.T)
before := append([]byte(nil), body...)
artifact := &domain.Artifact{Body: body}
result, err := validator.Validate(context.Background(), artifact, domain.OutputContract{
result, err := prepareForTest(t, validator, domain.OutputContract{
ValidationMode: domain.ValidationJSON,
})
}).Validate(context.Background(), artifact)
if err != nil {
t.Fatalf("validate JSON: %v", err)
}
@@ -213,10 +222,10 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
v := NewStandardValidator(tmp)
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -303,10 +312,10 @@ func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
v := NewStandardValidator(tmp)
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: filepath.Join("dnd", "schema.json"),
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -318,7 +327,7 @@ func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) {
v := NewStandardValidator(t.TempDir())
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
_, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: filepath.Join("dnd", "missing.json"),
})
@@ -343,10 +352,10 @@ func TestStandardValidatorJSONSchemaFailure(t *testing.T) {
v := NewStandardValidator(tmp)
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -361,7 +370,7 @@ func TestStandardValidatorJSONSchemaFailure(t *testing.T) {
func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) {
v := NewStandardValidator(t.TempDir())
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
_, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "missing.json",
})
@@ -377,7 +386,7 @@ func TestStandardValidatorJSONSchemaCompilationError(t *testing.T) {
}
v := NewStandardValidator(tmp)
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
res, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
@@ -398,10 +407,10 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
}`)},
}, "schemas")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -472,10 +481,10 @@ func TestFSValidatorEscapesSchemaResourcePath(t *testing.T) {
"schemas/" + schemaName: &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
}, "schemas")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: schemaName,
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)})
if err != nil || !res.IsValid {
t.Fatalf("validate schema %q: result=%#v error=%v", schemaName, res, err)
}
@@ -521,9 +530,9 @@ func TestSchemaReferencesPreserveEscapedFilenames(t *testing.T) {
}
}
func assertSchemaValidation(t *testing.T, validator Validator, schemaPath string) {
func assertSchemaValidation(t *testing.T, validator ValidationPreparer, schemaPath string) {
t.Helper()
result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`2`)}, domain.OutputContract{
result, err := validator.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: schemaPath,
})
@@ -544,10 +553,10 @@ func TestFSValidatorJSONSchemaPathContainment(t *testing.T) {
}`)},
}, "schemas")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "nested/events.schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -573,7 +582,7 @@ func TestFSValidatorJSONSchemaPathContainment(t *testing.T) {
"schemas/outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
}, "schemas")
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
_, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: tc.schemaPath,
})
@@ -598,10 +607,10 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
}`)},
}, "events.schema.json")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
res, err := prepareForTest(t, v, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "events.schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -609,7 +618,7 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{
_, err = v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "other.schema.json",
})
@@ -654,11 +663,15 @@ func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
}
v := NewStandardValidator(root)
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
plan, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "root.json",
})
if tc.wantError == "" {
if err != nil {
t.Fatalf("prepare schema: %v", err)
}
result, err := plan.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)})
if err != nil || !result.IsValid {
t.Fatalf("expected contained reference to validate, got result=%#v error=%v", result, err)
}
@@ -695,11 +708,15 @@ func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
"schemas/child.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
"outside.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
}, "schemas")
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
plan, err := v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "root.json",
})
if tc.wantError == "" {
if err != nil {
t.Fatalf("prepare schema: %v", err)
}
result, err := plan.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)})
if err != nil || !result.IsValid {
t.Fatalf("expected supported reference to validate, got result=%#v error=%v", result, err)
}
@@ -780,10 +797,10 @@ func TestJSONSchemaNumericConstraintsRetainJSONPrecision(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
validator := source.new(t, []byte(tc.schema))
result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.instance)}, domain.OutputContract{
result, err := prepareForTest(t, validator, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(tc.instance)})
if err != nil {
t.Fatalf("validate JSON Schema instance: %v", err)
}
@@ -801,7 +818,7 @@ func TestJSONSchemaDecodingRequiresOneCompleteDocument(t *testing.T) {
t.Run(source.name, func(t *testing.T) {
for _, schema := range []string{`{"type":`, `{} {}`} {
validator := source.new(t, []byte(schema))
_, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`1`)}, domain.OutputContract{
_, err := validator.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
@@ -812,10 +829,10 @@ func TestJSONSchemaDecodingRequiresOneCompleteDocument(t *testing.T) {
validator := source.new(t, []byte(`{}`))
for _, instance := range []string{`{"value":`, `1 2`} {
result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(instance)}, domain.OutputContract{
result, err := prepareForTest(t, validator, domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
}).Validate(context.Background(), &domain.Artifact{Body: []byte(instance)})
if err != nil {
t.Fatalf("instance %q: expected completed validation, got %v", instance, err)
}
@@ -844,11 +861,7 @@ func TestPreparedSchemaDocumentsRetainExactNumbers(t *testing.T) {
for _, source := range jsonSchemaValidatorSources() {
t.Run(source.name, func(t *testing.T) {
validator := source.new(t, []byte(schema))
preparer, ok := validator.(ValidationPreparer)
if !ok {
t.Fatal("validator does not support validation preparation")
}
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
prepared, err := validator.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
@@ -874,14 +887,14 @@ func TestPreparedSchemaDocumentsRetainExactNumbers(t *testing.T) {
type jsonSchemaValidatorSource struct {
name string
new func(*testing.T, []byte) Validator
new func(*testing.T, []byte) ValidationPreparer
}
func jsonSchemaValidatorSources() []jsonSchemaValidatorSource {
return []jsonSchemaValidatorSource{
{
name: "operating system files",
new: func(t *testing.T, schema []byte) Validator {
new: func(t *testing.T, schema []byte) ValidationPreparer {
t.Helper()
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "schema.json"), schema, 0o644); err != nil {
@@ -892,7 +905,7 @@ func jsonSchemaValidatorSources() []jsonSchemaValidatorSource {
},
{
name: "fs.FS",
new: func(t *testing.T, schema []byte) Validator {
new: func(t *testing.T, schema []byte) ValidationPreparer {
t.Helper()
return NewFSValidator(fstest.MapFS{
"schema.json": &fstest.MapFile{Data: schema},
@@ -914,6 +927,7 @@ func BenchmarkJSONValidation(b *testing.B) {
}
validator := NewStandardValidator("")
contract := domain.OutputContract{ValidationMode: domain.ValidationJSON}
plan := prepareForTest(b, validator, contract)
for _, benchmark := range benchmarks {
b.Run(benchmark.name, func(b *testing.B) {
@@ -922,7 +936,7 @@ func BenchmarkJSONValidation(b *testing.B) {
b.SetBytes(int64(len(benchmark.body)))
b.ResetTimer()
for range b.N {
result, err := validator.Validate(context.Background(), artifact, contract)
result, err := plan.Validate(context.Background(), artifact)
if err != nil || !result.IsValid {
b.Fatalf("validate JSON: result=%+v error=%v", result, err)
}
@@ -950,11 +964,12 @@ func BenchmarkBasicValidation(b *testing.B) {
validator := NewStandardValidator("")
contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
ctx := context.Background()
plan := prepareForTest(b, validator, contract)
for _, benchmark := range benchmarks {
b.Run(benchmark.name, func(b *testing.B) {
artifact := &domain.Artifact{Body: benchmark.body}
result, err := validator.Validate(ctx, artifact, contract)
result, err := plan.Validate(ctx, artifact)
if err != nil || result.IsValid != benchmark.wantValid {
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
}
@@ -964,7 +979,7 @@ func BenchmarkBasicValidation(b *testing.B) {
// scanning the body, so report latency rather than bytes per second.
b.ResetTimer()
for range b.N {
result, err := validator.Validate(ctx, artifact, contract)
result, err := plan.Validate(ctx, artifact)
if err != nil || result.IsValid != benchmark.wantValid {
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
}
@@ -998,7 +1013,7 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schema.json": &fstest.MapFile{Data: data},
}, ".")
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
_, err = v.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})