Prepare both smoke output validators
This commit is contained in:
@@ -208,6 +208,11 @@ final validation without adding transitional code or an intermediate gate.
|
|||||||
|
|
||||||
## Stage 7: Use Prepared Plans For Both Smoke Output Checks
|
## Stage 7: Use Prepared Plans For Both Smoke Output Checks
|
||||||
|
|
||||||
|
**Completed:** Smoke inference prepares separate JSON and schema plans and
|
||||||
|
checks syntax before schema matching. Go 1.27.3 `go test ./internal/usecase
|
||||||
|
-run 'Smoke|Diagnostic'` passed, including malformed, scalar, matching,
|
||||||
|
nonmatching, cancellation, and dispatch cases.
|
||||||
|
|
||||||
**Finding:** 4. **Files:** `internal/usecase/smoke_inference_case.go` and diagnostic tests.
|
**Finding:** 4. **Files:** `internal/usecase/smoke_inference_case.go` and diagnostic tests.
|
||||||
|
|
||||||
1. Replace the smoke case's `jsonValidator validate.Validator` with a
|
1. Replace the smoke case's `jsonValidator validate.Validator` with a
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ type smokeInferenceCase struct {
|
|||||||
kind domain.SmokeTestKind
|
kind domain.SmokeTestKind
|
||||||
request domain.GenerateRequest
|
request domain.GenerateRequest
|
||||||
expectedContent string
|
expectedContent string
|
||||||
jsonValidator validate.Validator
|
jsonValidation validate.PreparedValidation
|
||||||
schemaValidation validate.PreparedValidation
|
schemaValidation validate.PreparedValidation
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,6 +52,12 @@ func prepareSmokeInferenceCase(
|
|||||||
}, nil
|
}, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
validator := validate.NewFSValidator(smokeSchemaFS, ".")
|
||||||
|
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
|
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
|
||||||
|
}
|
||||||
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
|
||||||
Format: domain.FormatJSON,
|
Format: domain.FormatJSON,
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
@@ -64,7 +70,7 @@ func prepareSmokeInferenceCase(
|
|||||||
return &smokeInferenceCase{
|
return &smokeInferenceCase{
|
||||||
kind: kind,
|
kind: kind,
|
||||||
expectedContent: smokeStructuredExpectedContent,
|
expectedContent: smokeStructuredExpectedContent,
|
||||||
jsonValidator: validate.NewStandardValidator(""),
|
jsonValidation: jsonPlan,
|
||||||
schemaValidation: plan,
|
schemaValidation: plan,
|
||||||
request: newGenerationRequest(
|
request: newGenerationRequest(
|
||||||
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
|
||||||
@@ -97,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene
|
|||||||
return facts, nil
|
return facts, nil
|
||||||
case domain.SmokeTestStructuredOutput:
|
case domain.SmokeTestStructuredOutput:
|
||||||
artifact := domain.Artifact{Body: []byte(response.Content)}
|
artifact := domain.Artifact{Body: []byte(response.Content)}
|
||||||
jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
|
jsonResult, err := c.jsonValidation.Validate(ctx, &artifact)
|
||||||
Format: domain.FormatJSON,
|
|
||||||
ValidationMode: domain.ValidationJSON,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package usecase
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -102,6 +103,7 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
|
{name: "matches", content: caseValue.expectedContent, wantValidJSON: true, wantSchemaMatched: boolPointer(true)},
|
||||||
{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
{name: "valid JSON misses schema", content: `{"status":"other"}`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
|
{name: "scalar JSON misses schema", content: `42`, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
{name: "additional property misses schema", content: additionalPropertyContent, wantValidJSON: true, wantSchemaMatched: boolPointer(false)},
|
||||||
{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
|
{name: "invalid JSON", content: `{invalid`, wantSchemaMatched: nil},
|
||||||
{name: "empty content", content: "", wantSchemaMatched: nil},
|
{name: "empty content", content: "", wantSchemaMatched: nil},
|
||||||
@@ -125,3 +127,19 @@ func TestSmokeStructuredInferenceCaseUsesPrivatePreparedSchema(t *testing.T) {
|
|||||||
t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
|
t.Fatalf("facts were populated without generated content: %+v", noResponseFacts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSmokeStructuredInferenceCaseHonorsCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := prepareSmokeInferenceCase(ctx, domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("prepare error = %v, want cancellation", err)
|
||||||
|
}
|
||||||
|
caseValue, err := prepareSmokeInferenceCase(context.Background(), domain.SmokeTestStructuredOutput, domain.SmokeTestTarget{}, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
facts, err := caseValue.evaluate(ctx, &domain.GenerateResponse{Content: `{"status":"ok"}`})
|
||||||
|
if !errors.Is(err, context.Canceled) || facts.ValidJSON != nil || facts.SchemaMatched != nil {
|
||||||
|
t.Fatalf("evaluation facts = %+v, error = %v", facts, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user