base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Prepare both smoke output validators

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-05 20:00:33 +00:00
parent d78474a3bf
commit 4b4ce43796
3 changed files with 32 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
internal/usecase/smoke_inference_case.go
View File

@@ -25,7 +25,7 @@ type smokeInferenceCase struct {
kind domain.SmokeTestKind
request domain.GenerateRequest
expectedContent string
jsonValidator validate.Validator
jsonValidation validate.PreparedValidation
schemaValidation validate.PreparedValidation
}
@@ -52,6 +52,12 @@ func prepareSmokeInferenceCase(
}, nil
case domain.SmokeTestStructuredOutput:
validator := validate.NewFSValidator(smokeSchemaFS, ".")
jsonPlan, err := validator.PrepareValidation(ctx, domain.OutputContract{
Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON,
})
if err != nil {
return nil, fmt.Errorf("%w: prepare private JSON validation: %w", ErrValidation, err)
}
plan, err := validator.PrepareValidation(ctx, domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema,
@@ -64,7 +70,7 @@ func prepareSmokeInferenceCase(
return &smokeInferenceCase{
kind: kind,
expectedContent: smokeStructuredExpectedContent,
jsonValidator: validate.NewStandardValidator(""),
jsonValidation: jsonPlan,
schemaValidation: plan,
request: newGenerationRequest(
domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: smokeStructuredPromptPrefix + smokeStructuredExpectedContent}}},
@@ -97,10 +103,7 @@ func (c *smokeInferenceCase) evaluate(ctx context.Context, response *domain.Gene
return facts, nil
case domain.SmokeTestStructuredOutput:
artifact := domain.Artifact{Body: []byte(response.Content)}
jsonResult, err := c.jsonValidator.Validate(ctx, &artifact, domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSON,
})
jsonResult, err := c.jsonValidation.Validate(ctx, &artifact)
if err != nil {
return facts, fmt.Errorf("%w: evaluate private smoke-test JSON: %w", ErrValidation, err)
}