From 9463c9463a8c3e7bf554fc80a79ba11af20d5833 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 5 Nov 2026 20:04:55 +0000 Subject: [PATCH] Migrate validator tests to prepared plans --- docs/roadmap/implementation.md | 6 + internal/usecase/runner_test.go | 17 +-- internal/validate/standard_validator_test.go | 121 +++++++++++-------- 3 files changed, 80 insertions(+), 64 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 9049348..d2a0b22 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -250,6 +250,12 @@ temporarily fail to compile; migrate them in Stage 9, not through a wrapper. ## Stage 9: Migrate Validator Tests To Prepared Execution +**Completed:** Validator tests prepare plans before execution, distinguish +preparation failures from invalid instances, and retain host/FS, portability, +precision, freezing, and synchronous-cancellation coverage. The retired mock +interface was removed. Go 1.27.3 `go test ./internal/validate ./internal/usecase` +and `go test . -run 'Schema|Smoke'` passed. + **Finding:** 4. **Scope:** `internal/validate` tests, particularly standard-validator and cancellation cases. 1. Express meaningful direct-validation cases as plan preparation followed by diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index d335574..b534df7 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -240,14 +240,6 @@ type fakeValidator struct { validateCalls int } -func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { - f.validateCalls++ - if f.err != nil { - return domain.ValidationResult{}, f.err - } - return f.result, nil -} - func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.OutputContract) (validate.PreparedValidation, error) { var schemaDocument any if contract.ValidationMode == domain.ValidationJSONSchema { @@ -261,17 +253,20 @@ func (f *fakeValidator) PrepareValidation(_ context.Context, contract domain.Out schemaDocument = map[string]any{"type": "object"} } } - return &fakePreparedValidator{validator: f, contract: contract, schemaDocument: schemaDocument}, nil + return &fakePreparedValidator{validator: f, schemaDocument: schemaDocument}, nil } type fakePreparedValidator struct { validator *fakeValidator - contract domain.OutputContract schemaDocument any } func (p *fakePreparedValidator) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) { - return p.validator.Validate(ctx, artifact, p.contract) + p.validator.validateCalls++ + if p.validator.err != nil { + return domain.ValidationResult{}, p.validator.err + } + return p.validator.result, nil } func (p *fakePreparedValidator) SchemaDocument() any { diff --git a/internal/validate/standard_validator_test.go b/internal/validate/standard_validator_test.go index b094920..704a4ba 100644 --- a/internal/validate/standard_validator_test.go +++ b/internal/validate/standard_validator_test.go @@ -17,12 +17,21 @@ import ( "go.maximumdirect.net/promptkit/internal/domain" ) +func prepareForTest(t testing.TB, preparer ValidationPreparer, contract domain.OutputContract) PreparedValidation { + t.Helper() + plan, err := preparer.PrepareValidation(context.Background(), contract) + if err != nil { + t.Fatalf("prepare validation: %v", err) + } + return plan +} + func TestStandardValidatorNoneSkipped(t *testing.T) { v := NewStandardValidator("") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationNone, - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -44,9 +53,9 @@ func TestStandardValidatorBasicSuccess(t *testing.T) { {name: "invalid UTF-8 byte", body: []byte{0xff}}, } { t.Run(tc.name, func(t *testing.T) { - res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationBasic, - }) + }).Validate(context.Background(), &domain.Artifact{Body: tc.body}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -68,9 +77,9 @@ func TestStandardValidatorBasicFailureEmpty(t *testing.T) { {name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")}, } { t.Run(tc.name, func(t *testing.T) { - res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationBasic, - }) + }).Validate(context.Background(), &domain.Artifact{Body: tc.body}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -110,7 +119,7 @@ func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)}, tc.contract) + res, err := prepareForTest(t, v, tc.contract).Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)}) if err != nil { t.Fatalf("validate: %v", err) } @@ -124,9 +133,9 @@ func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) { func TestStandardValidatorJSONSuccess(t *testing.T) { v := NewStandardValidator("") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSON, - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -138,9 +147,9 @@ func TestStandardValidatorJSONSuccess(t *testing.T) { func TestStandardValidatorJSONFailure(t *testing.T) { v := NewStandardValidator("") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSON, - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -175,9 +184,9 @@ func TestJSONValidationChecksCompleteSyntaxWithoutChangingArtifact(t *testing.T) before := append([]byte(nil), body...) artifact := &domain.Artifact{Body: body} - result, err := validator.Validate(context.Background(), artifact, domain.OutputContract{ + result, err := prepareForTest(t, validator, domain.OutputContract{ ValidationMode: domain.ValidationJSON, - }) + }).Validate(context.Background(), artifact) if err != nil { t.Fatalf("validate JSON: %v", err) } @@ -213,10 +222,10 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) { v := NewStandardValidator(tmp) - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -303,10 +312,10 @@ func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) { v := NewStandardValidator(tmp) - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: filepath.Join("dnd", "schema.json"), - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -318,7 +327,7 @@ func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) { func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) { v := NewStandardValidator(t.TempDir()) - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + _, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: filepath.Join("dnd", "missing.json"), }) @@ -343,10 +352,10 @@ func TestStandardValidatorJSONSchemaFailure(t *testing.T) { v := NewStandardValidator(tmp) - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -361,7 +370,7 @@ func TestStandardValidatorJSONSchemaFailure(t *testing.T) { func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) { v := NewStandardValidator(t.TempDir()) - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{ + _, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "missing.json", }) @@ -377,7 +386,7 @@ func TestStandardValidatorJSONSchemaCompilationError(t *testing.T) { } v := NewStandardValidator(tmp) - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + res, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", }) @@ -398,10 +407,10 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) { }`)}, }, "schemas") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "events.schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -472,10 +481,10 @@ func TestFSValidatorEscapesSchemaResourcePath(t *testing.T) { "schemas/" + schemaName: &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, }, "schemas") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: schemaName, - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}) if err != nil || !res.IsValid { t.Fatalf("validate schema %q: result=%#v error=%v", schemaName, res, err) } @@ -521,9 +530,9 @@ func TestSchemaReferencesPreserveEscapedFilenames(t *testing.T) { } } -func assertSchemaValidation(t *testing.T, validator Validator, schemaPath string) { +func assertSchemaValidation(t *testing.T, validator ValidationPreparer, schemaPath string) { t.Helper() - result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`2`)}, domain.OutputContract{ + result, err := validator.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: schemaPath, }) @@ -544,10 +553,10 @@ func TestFSValidatorJSONSchemaPathContainment(t *testing.T) { }`)}, }, "schemas") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "nested/events.schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -573,7 +582,7 @@ func TestFSValidatorJSONSchemaPathContainment(t *testing.T) { "schemas/outside.schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, }, "schemas") - _, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + _, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: tc.schemaPath, }) @@ -598,10 +607,10 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) { }`)}, }, "events.schema.json") - res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + res, err := prepareForTest(t, v, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "events.schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -609,7 +618,7 @@ func TestFSValidatorSingleSchemaFileUsesBaseName(t *testing.T) { t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid) } - _, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"events":[]}`)}, domain.OutputContract{ + _, err = v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "other.schema.json", }) @@ -654,11 +663,15 @@ func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) { } v := NewStandardValidator(root) - result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{ + plan, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "root.json", }) if tc.wantError == "" { + if err != nil { + t.Fatalf("prepare schema: %v", err) + } + result, err := plan.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}) if err != nil || !result.IsValid { t.Fatalf("expected contained reference to validate, got result=%#v error=%v", result, err) } @@ -695,11 +708,15 @@ func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) { "schemas/child.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)}, "outside.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)}, }, "schemas") - result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{ + plan, err := v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "root.json", }) if tc.wantError == "" { + if err != nil { + t.Fatalf("prepare schema: %v", err) + } + result, err := plan.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}) if err != nil || !result.IsValid { t.Fatalf("expected supported reference to validate, got result=%#v error=%v", result, err) } @@ -780,10 +797,10 @@ func TestJSONSchemaNumericConstraintsRetainJSONPrecision(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { validator := source.new(t, []byte(tc.schema)) - result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.instance)}, domain.OutputContract{ + result, err := prepareForTest(t, validator, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(tc.instance)}) if err != nil { t.Fatalf("validate JSON Schema instance: %v", err) } @@ -801,7 +818,7 @@ func TestJSONSchemaDecodingRequiresOneCompleteDocument(t *testing.T) { t.Run(source.name, func(t *testing.T) { for _, schema := range []string{`{"type":`, `{} {}`} { validator := source.new(t, []byte(schema)) - _, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(`1`)}, domain.OutputContract{ + _, err := validator.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", }) @@ -812,10 +829,10 @@ func TestJSONSchemaDecodingRequiresOneCompleteDocument(t *testing.T) { validator := source.new(t, []byte(`{}`)) for _, instance := range []string{`{"value":`, `1 2`} { - result, err := validator.Validate(context.Background(), &domain.Artifact{Body: []byte(instance)}, domain.OutputContract{ + result, err := prepareForTest(t, validator, domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", - }) + }).Validate(context.Background(), &domain.Artifact{Body: []byte(instance)}) if err != nil { t.Fatalf("instance %q: expected completed validation, got %v", instance, err) } @@ -844,11 +861,7 @@ func TestPreparedSchemaDocumentsRetainExactNumbers(t *testing.T) { for _, source := range jsonSchemaValidatorSources() { t.Run(source.name, func(t *testing.T) { validator := source.new(t, []byte(schema)) - preparer, ok := validator.(ValidationPreparer) - if !ok { - t.Fatal("validator does not support validation preparation") - } - prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{ + prepared, err := validator.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", }) @@ -874,14 +887,14 @@ func TestPreparedSchemaDocumentsRetainExactNumbers(t *testing.T) { type jsonSchemaValidatorSource struct { name string - new func(*testing.T, []byte) Validator + new func(*testing.T, []byte) ValidationPreparer } func jsonSchemaValidatorSources() []jsonSchemaValidatorSource { return []jsonSchemaValidatorSource{ { name: "operating system files", - new: func(t *testing.T, schema []byte) Validator { + new: func(t *testing.T, schema []byte) ValidationPreparer { t.Helper() root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "schema.json"), schema, 0o644); err != nil { @@ -892,7 +905,7 @@ func jsonSchemaValidatorSources() []jsonSchemaValidatorSource { }, { name: "fs.FS", - new: func(t *testing.T, schema []byte) Validator { + new: func(t *testing.T, schema []byte) ValidationPreparer { t.Helper() return NewFSValidator(fstest.MapFS{ "schema.json": &fstest.MapFile{Data: schema}, @@ -914,6 +927,7 @@ func BenchmarkJSONValidation(b *testing.B) { } validator := NewStandardValidator("") contract := domain.OutputContract{ValidationMode: domain.ValidationJSON} + plan := prepareForTest(b, validator, contract) for _, benchmark := range benchmarks { b.Run(benchmark.name, func(b *testing.B) { @@ -922,7 +936,7 @@ func BenchmarkJSONValidation(b *testing.B) { b.SetBytes(int64(len(benchmark.body))) b.ResetTimer() for range b.N { - result, err := validator.Validate(context.Background(), artifact, contract) + result, err := plan.Validate(context.Background(), artifact) if err != nil || !result.IsValid { b.Fatalf("validate JSON: result=%+v error=%v", result, err) } @@ -950,11 +964,12 @@ func BenchmarkBasicValidation(b *testing.B) { validator := NewStandardValidator("") contract := domain.OutputContract{ValidationMode: domain.ValidationBasic} ctx := context.Background() + plan := prepareForTest(b, validator, contract) for _, benchmark := range benchmarks { b.Run(benchmark.name, func(b *testing.B) { artifact := &domain.Artifact{Body: benchmark.body} - result, err := validator.Validate(ctx, artifact, contract) + result, err := plan.Validate(ctx, artifact) if err != nil || result.IsValid != benchmark.wantValid { b.Fatalf("validate basic output: result=%+v error=%v", result, err) } @@ -964,7 +979,7 @@ func BenchmarkBasicValidation(b *testing.B) { // scanning the body, so report latency rather than bytes per second. b.ResetTimer() for range b.N { - result, err := validator.Validate(ctx, artifact, contract) + result, err := plan.Validate(ctx, artifact) if err != nil || result.IsValid != benchmark.wantValid { b.Fatalf("validate basic output: result=%+v error=%v", result, err) } @@ -998,7 +1013,7 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) { v := NewFSValidator(fstest.MapFS{ "schema.json": &fstest.MapFile{Data: data}, }, ".") - _, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{ + _, err = v.PrepareValidation(context.Background(), domain.OutputContract{ ValidationMode: domain.ValidationJSONSchema, SchemaPath: "schema.json", })