Migrate combat semantics validation to native execution and unavailable assessments

This commit is contained in:
2026-11-22 00:42:20 +00:00
parent 52a1c96327
commit 5d3696d4a0
13 changed files with 571 additions and 493 deletions

View File

@@ -1,24 +1,58 @@
package combat_semantics
package assets_test
import (
"bytes"
"context"
"encoding/json"
"gitea.maximumdirect.net/eric/notarius/assets"
"io/fs"
"strings"
"testing"
"go.maximumdirect.net/promptkit"
"github.com/santhosh-tekuri/jsonschema/v6"
)
func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
schema, err := loadResponseSchema()
func TestCombatSemanticsPromptKeepsItsSceneLocalPolicyAndInput(t *testing.T) {
_, engine := loadNativeAssets(t)
const transcript = `{"sentinel":"combat-semantics-transcript"}`
handle, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only",
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(transcript)},
})
if err != nil {
t.Fatal(err)
}
defer handle.Discard()
details := handle.Details()
if details.SelectedProfileID != "notarius/dnd-extraction" || details.StructuredOutput == nil {
t.Fatal("native validator defaults/output contract lost")
}
var all string
for _, message := range details.Messages {
all += message.Content
if strings.Contains(message.Content, transcript) && (message.CacheControl == nil || message.CacheControl.Type != promptkit.CacheControlEphemeral) {
t.Fatal("chunk transcript native cache hint lost")
}
}
for _, content := range []string{"substantive active combat materially organizes the", "Do not judge the scene title", transcript} {
if strings.Count(all, content) != 1 {
t.Fatalf("scene-local context not rendered once: %q", content)
}
}
if _, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "notarius/dnd.scene_descriptions.validate_combat", APIKey: "test-only"}); err == nil {
t.Fatal("native validator accepted missing transcript")
}
}
func TestCombatSemanticsResponseSchemaRetainsClassificationAndExplanationRules(t *testing.T) {
schema, err := fs.ReadFile(assets.FS(), "dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json")
if err != nil {
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
}
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema)
}
for _, classification := range []string{"combat", "non_combat"} {
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil {
if err := validateCombatSemanticsSchema(map[string]any{"classification": classification, "explanation": "The chunk contains sustained attack exchanges."}, schema); err != nil {
t.Fatalf("valid %q classification rejected: %v", classification, err)
}
}
@@ -36,29 +70,11 @@ func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) {
{name: "oversized explanation", response: map[string]any{"classification": "combat", "explanation": strings.Repeat("a", 513)}},
} {
t.Run(test.name, func(t *testing.T) {
if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil {
if err := validateCombatSemanticsSchema(test.response, schema); err == nil {
t.Fatal("invalid private response accepted")
}
})
}
if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) {
t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema)
}
}
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
first, err := loadResponseSchema()
if err != nil {
t.Fatal(err)
}
first.JSONSchema[0] = '['
second, err := loadResponseSchema()
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
}
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
}
}
func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error {

View File

@@ -764,6 +764,29 @@ units, full-source fallback, final-turn acceptance, and cancellation.
## Stage 17: Migrate The Combat-Semantics LLM Validator
**Status:** Complete. The validator now makes one native request on the shared
engine, preserving the validator profile/default, session, execution overrides,
transient-retry override, observer, and preparation callback. Static prompt
dependencies replace asset/schema registration and manifest/fingerprint metadata.
Native errors and failed validation status are checked before decoding; execution
and interpretation failures produce an unavailable assessment with inspectable
causes, not semantic rejection or invented correction guidance. Cancellation
does not deliver a judgment. Successful classification retains the original
combat policy, scene-local transcript boundary, explanation checks, and actionable
producer guidance. Interpretation and its existing cases are independently
testable; evaluation fixtures and schema/policy assets are unchanged.
Added native provider tests for judgments, override/input isolation, nil-error
failed validation, unavailable provider/capacity, interpretation failure,
cancellation, and nested same-backend admission. A runnable framework contract
proves complete ordered native assessment on a backend limited to one active call,
without holding producer admission or repeating the final chain.
Asset tests/vet/race, independent interpretation/evaluation tests/vet/race,
focused pipeline tests/vet/race, and `git diff --check` pass. The concrete
validator package remains blocked at setup by shared legacy asset helpers
importing old ConfigKit's removed YAML `plugin/limit`; its native runtime tests
are not claimed passing. Scheduled dependency cleanup and final verification
must execute them. No validator-owned semantic continuation loop was added.
**Goal:** validators use native execution without another semantic retry loop.
**Inspect/change:** `internal/modules/dnd/validate/scenedescriptions/combat_semantics`,

View File

@@ -0,0 +1,58 @@
package pipeline
import (
"context"
"fmt"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestNativeAssessmentChainSharesOneLimitedBackendWithoutHoldingProducerAdmission(t *testing.T) {
calls := 0
engine := nativeProducerEngine(t, producerClientFunc(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if request.Target.BackendID != "limited" {
t.Fatal("fixture did not select the shared backend")
}
calls++
return &promptkit.GenerateResponse{Content: `{"value":1}`}, nil
}), promptkit.SettingsInput(configkit.Bytes("shared assessment backend", []byte(`backends:
- id: limited
endpoint: https://example.invalid/v1
concurrency_limit: 1
queue_capacity: 0
profiles:
- id: base
backend: limited
model: assessment-model
api_key_required: false
`))))
chain := preparedValidatorChain{}
for _, name := range []string{"first-native", "second-native"} {
chain.validators = append(chain.validators, preparedValidator{resolved: ResolvedValidator{Binding: Binding(name)}})
}
var order []string
operation := contracts.ProducerOperation[int]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
Assess: func(ctx context.Context, _ int) (contracts.CandidateAssessment, error) {
return executeValidationChain(ctx, chain, func(ctx context.Context, validator preparedValidator) (contracts.ValidationResult, error) {
order = append(order, validator.resolved.Binding.Module)
result, err := engine.Run(ctx, nativeProducerInitial())
if err != nil {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: err}
}
if !result.Validation.IsValid {
return contracts.ValidationResult{}, &contracts.ValidationUnavailableError{Cause: fmt.Errorf("native contract failed")}
}
return contracts.ValidationResult{Approved: true}, nil
})
},
}
value, outcome, err := RunProducer(context.Background(), engine, operation, nativeProducerInitial(), interpretProducerNumber, nil, nil)
if err != nil || value != 1 || outcome.Disposition != contracts.ProducerAccepted || calls != 3 || !reflect.DeepEqual(order, []string{"first-native", "second-native"}) {
t.Fatalf("nested assessment=%d %#v %v, native calls=%d order=%v", value, outcome, err, calls, order)
}
}

View File

@@ -22,7 +22,7 @@ func (f producerClientFunc) Generate(ctx context.Context, request promptkit.Gene
return f(ctx, request)
}
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
func nativeProducerEngine(t *testing.T, client promptkit.LLMClient, extraInputs ...configkit.Input) *promptkit.Engine {
t.Helper()
settings := `profiles:
- id: base
@@ -52,8 +52,12 @@ prompts:
- {role: user, content: 'Correct: {{input "guidance"}}'}
output: {format: json, validation_mode: json, repair_attempts: 0}
`
underlays := []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}}
if len(extraInputs) > 0 {
underlays = append(underlays, configkit.Underlay{Inputs: extraInputs})
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("producer fixture", []byte(settings)))}}},
Underlays: underlays,
}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)

View File

@@ -0,0 +1,61 @@
package combat_semantics
import (
"encoding/json"
"errors"
"io"
"os"
"strings"
"testing"
)
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
content, err := os.ReadFile("testdata/evaluation_cases.json")
if err != nil {
t.Fatal(err)
}
var cases []struct {
Name string `json:"name"`
TranscriptUnits []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"transcript_units"`
ProposedKind string `json:"proposed_kind"`
ExpectedClassification string `json:"expected_classification"`
ReviewerRationale string `json:"reviewer_rationale"`
}
decoder := json.NewDecoder(strings.NewReader(string(content)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&cases); err != nil {
t.Fatalf("decode evaluation cases: %v", err)
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
}
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
for _, value := range cases {
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
t.Fatalf("invalid evaluation case: %#v", value)
}
seenNames[value.Name] = true
switch value.ProposedKind {
case "combat", "narrative", "recap", "meta":
default:
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
}
switch value.ExpectedClassification {
case "combat", "non_combat":
seenClassifications[value.ExpectedClassification] = true
default:
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
}
for _, unit := range value.TranscriptUnits {
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
t.Fatalf("invalid transcript unit: %#v", unit)
}
}
}
if len(seenClassifications) != 2 {
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
}
}

View File

@@ -0,0 +1,70 @@
package combat_semantics
import (
"fmt"
"strings"
"unicode/utf8"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
const (
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
maximumExplanationRunes = 512
)
type classificationResponse struct {
Classification string `json:"classification"`
Explanation string `json:"explanation"`
}
func interpretResponse(response classificationResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
explanation, err := validateExplanation(response.Explanation)
if err != nil {
return contracts.ValidationResult{}, err
}
switch response.Classification {
case "combat":
if proposedKind == dnd.SceneKindCombat {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeActiveCombatNotClassified,
Message: "The current chunk contains substantive active combat that is not classified as combat.",
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
}, nil
case "non_combat":
if proposedKind != dnd.SceneKindCombat {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeCombatClassificationUnsupported,
Message: "The current chunk does not support a combat classification.",
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
}, nil
default:
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
}
}
func validateExplanation(value string) (string, error) {
if !utf8.ValidString(value) {
return "", validatorErrorf("validator explanation must be valid UTF-8")
}
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return "", validatorErrorf("validator explanation must not be blank")
}
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
}
return trimmed, nil
}
func validatorErrorf(format string, args ...any) error {
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
}

View File

@@ -0,0 +1,53 @@
package combat_semantics
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
)
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
for _, test := range []struct {
name string
kind dnd.SceneKind
response classificationResponse
approved bool
reasonCode string
guidance string
correction string
wantError string
}{
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
{name: "approve combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
{name: "add combat", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
{name: "remove combat", kind: dnd.SceneKindCombat, response: classificationResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: classificationResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
} {
t.Run(test.name, func(t *testing.T) {
result, err := interpretResponse(test.response, test.kind)
if test.wantError != "" {
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
}
return
}
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
t.Fatalf("Validate() = %#v, %v", result, err)
}
if test.correction != "" && result.CorrectionGuidance != test.correction {
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
}
if !result.Approved && strings.Contains(result.CorrectionGuidance, result.ReasonCode) {
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
}
})
}
}

View File

@@ -0,0 +1,144 @@
package combat_semantics
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/promptkit"
)
func TestNativeValidatorReturnsOnlyCompletedSemanticJudgments(t *testing.T) {
for _, classification := range []string{"combat", "non_combat"} {
for _, proposed := range []dnd.SceneKind{dnd.SceneKindCombat, dnd.SceneKindNarrative} {
request := validRequest(proposed)
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: classification, Explanation: "The supplied transcript supports this judgment."}}
result, err := newValidator(t, client).Validate(context.Background(), request)
wantApproval := (classification == "combat") == (proposed == dnd.SceneKindCombat)
if err != nil || result.Approved != wantApproval || len(client.requests) != 1 {
t.Fatalf("judgment=%#v %v", result, err)
}
if !wantApproval && (strings.TrimSpace(result.CorrectionGuidance) == "" || strings.Contains(result.CorrectionGuidance, result.ReasonCode)) {
t.Fatal("rejection lost actionable guidance or copied diagnostic labels")
}
}
}
}
func TestNativeFailuresAreUnavailableRatherThanRejections(t *testing.T) {
failure := errors.New("provider unavailable")
for _, test := range []struct {
name, raw string
failure error
}{
{"provider failure", "", failure},
{"invalid JSON", "{", nil},
{"invalid classification", `{"classification":"uncertain","explanation":"Uncertain."}`, nil},
{"uninterpretable explanation", `{"classification":"combat","explanation":" "}`, nil},
} {
t.Run(test.name, func(t *testing.T) {
client := &fakeCombatSemanticsClient{raw: test.raw, err: test.failure}
request := validRequest(dnd.SceneKindCombat)
prepared := 0
request.Execution.OnPrepared = func(context.Context, promptkit.PreparedRun) { prepared++ }
result, err := newValidator(t, client).Validate(context.Background(), request)
var unavailable *contracts.ValidationUnavailableError
if !errors.As(err, &unavailable) || result.Approved || result.ReasonCode != "" || result.CorrectionGuidance != "" || prepared != 1 {
t.Fatalf("unavailable assessment=%#v %v, native executions=%d", result, err, prepared)
}
if test.failure != nil && (!errors.Is(err, failure) || len(client.requests) != 1) {
t.Fatalf("provider failure was flattened or retried: %v", err)
}
})
}
}
func TestNilErrorFailedNativeValidationCannotBecomeApproval(t *testing.T) {
client := &fakeCombatSemanticsClient{raw: `{"classification":"combat"}`}
validator := newValidator(t, client)
request := validRequest(dnd.SceneKindCombat)
native, err := validator.engine.Run(context.Background(), promptkit.RunRequest{
PromptID: PromptID, ProfileID: request.LLMProfile,
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
})
if err != nil || native == nil || native.Validation.Status != promptkit.ValidationFailed || native.Validation.IsValid {
t.Fatalf("fixture did not exercise nil-error structural failure: %#v %v", native, err)
}
result, err := validator.Validate(context.Background(), request)
var unavailable *contracts.ValidationUnavailableError
if !errors.As(err, &unavailable) || !errors.Is(err, promptkit.ErrValidation) || result.Approved || result.CorrectionGuidance != "" {
t.Fatalf("failed native contract accepted/interpreted: %#v %v", result, err)
}
}
func TestNestedValidatorUsesTheReleasedNativeBackendAdmission(t *testing.T) {
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
validator := newValidator(t, client)
request := validRequest(dnd.SceneKindNarrative)
assessments := 0
operation := contracts.ProducerOperation[dnd.SceneDescriptionList]{
Controls: contracts.ProducerControls{Mode: contracts.ModeStrict}, ReuseEligible: true,
Assess: func(ctx context.Context, candidate dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
assessments++
nested := request
nested.Value = candidate
judgment, err := validator.Validate(ctx, nested)
if err != nil {
return contracts.CandidateAssessment{}, err
}
decision := contracts.AssessmentRejected
if judgment.Approved {
decision = contracts.AssessmentApproved
}
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: Key, Decision: decision, Result: judgment}}}, nil
},
}
_, outcome, err := pipeline.RunProducer(context.Background(), validator.engine, operation, promptkit.RunRequest{
PromptID: PromptID, ProfileID: request.LLMProfile,
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline(string(request.Chunk.Content))},
}, func(context.Context, *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) {
return &request.Value, nil, nil
}, nil, nil)
if err != nil || outcome.Disposition != contracts.ProducerAccepted || assessments != 1 || len(client.requests) != 2 {
t.Fatalf("nested native admission=%#v %v, assessments=%d calls=%d", outcome, err, assessments, len(client.requests))
}
for _, call := range client.requests {
if call.Target.BackendID != "validator-backend" {
t.Fatal("fixture did not exercise the same limited backend")
}
}
}
func TestCapacityFailureAndCancellationNeverBecomeSemanticGuidance(t *testing.T) {
t.Run("capacity unavailable", func(t *testing.T) {
var validator *Validator
var nestedErr error
request := validRequest(dnd.SceneKindNarrative)
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."}}
client.onGenerate = func(ctx context.Context, _ promptkit.GenerateRequest) {
// The outer fake provider still holds the single active native admission.
// The configured zero queue makes this a capacity error, not a waiting test.
_, nestedErr = validator.Validate(ctx, request)
}
validator = newValidator(t, client)
result, err := validator.Validate(context.Background(), request)
var unavailable *contracts.ValidationUnavailableError
if err != nil || !result.Approved || !errors.As(nestedErr, &unavailable) || !errors.Is(nestedErr, promptkit.ErrCapacityExceeded) || len(client.requests) != 1 {
t.Fatalf("capacity assessment=%#v %v, nested=%v", result, err, nestedErr)
}
})
t.Run("canceled delivery", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No encounter."},
onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
result, err := newValidator(t, client).Validate(ctx, validRequest(dnd.SceneKindNarrative))
if !errors.Is(err, context.Canceled) || result.Approved || result.CorrectionGuidance != "" || len(client.requests) != 1 {
t.Fatalf("cancellation=%#v %v", result, err)
}
})
}

View File

@@ -1,72 +0,0 @@
package combat_semantics
import (
"fmt"
"io/fs"
"sync"
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
)
const promptAssetRoot = "assets/prompts"
var promptAssetManifest = shared.PromptAssetManifest{
ModuleDir: PromptID,
ModuleFiles: []promptfs.ModulePromptFile{
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
{Name: "instructions.md", Path: "prompts/instructions.md"},
},
SharedFiles: []string{
"common-dnd-system.md",
"common-dnd-scene-combat-policy.md",
"common-dnd-transcript-chunk.md",
},
}
func moduleAssetFS() (fs.FS, error) {
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics")
if err != nil {
return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err)
}
return assets, nil
}
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
assets, err := moduleAssetFS()
if err != nil {
return err
}
promptFS, err := promptAssetManifest.PromptFS(assets)
if err != nil {
return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err)
}
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
return err
}
schemas, err := fs.Sub(assets, "schemas")
if err != nil {
return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err)
}
return registry.RegisterSchemaFS(schemas, ".")
}
func promptAssetMetadata() (string, error) {
promptAssetHashOnce.Do(func() {
assets, err := moduleAssetFS()
if err != nil {
promptAssetHashErr = err
return
}
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
})
return promptAssetHash, promptAssetHashErr
}
var (
promptAssetHashOnce sync.Once
promptAssetHash string
promptAssetHashErr error
)

View File

@@ -1,90 +0,0 @@
package combat_semantics
import (
"context"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) {
registry := llm.NewAssetRegistry()
if err := RegisterPromptAssets(registry); err != nil {
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
}
engine := newCombatSemanticsPromptEngine(t, registry)
inputs := map[string]promptkit.ArtifactRef{
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`),
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs,
})
if err != nil {
t.Fatalf("Prepare() error = %v, want nil", err)
}
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" {
t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared)
}
if len(prepared.Messages) != 4 {
t.Fatalf("prepared message count = %d, want 4", len(prepared.Messages))
}
for index, message := range prepared.Messages {
wantEphemeral := index == 3
gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral
if gotEphemeral != wantEphemeral {
t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral)
}
}
assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1)
assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 3)
for name := range inputs {
missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1)
for inputName, input := range inputs {
if inputName != name {
missing[inputName] = input
}
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing,
}); err == nil {
t.Fatalf("Prepare() without required %q input = nil error", name)
}
}
}
func TestPromptAssetMetadataIsContentSafe(t *testing.T) {
hash, err := promptAssetMetadata()
if err != nil || !strings.HasPrefix(hash, "sha256:") {
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
}
if strings.Contains(hash, "non_combat") {
t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash)
}
}
func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine {
t.Helper()
return promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
"combat-semantics-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "combat-semantics-test-model"),
})
}
func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) {
t.Helper()
index := -1
occurrences := 0
for messageIndex, message := range messages {
count := strings.Count(message.Content, sentinel)
if count > 0 {
index = messageIndex
occurrences += count
}
}
if occurrences != 1 || index != wantIndex {
t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex)
}
}

View File

@@ -1,25 +0,0 @@
package combat_semantics
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
const (
PromptID = "dnd.scene_descriptions.validate_combat"
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm")
ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm"
ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1"
SchemaVersion = "v1"
)
func loadResponseSchema() (llm.ResponseSchema, error) {
assets, err := moduleAssetFS()
if err != nil {
return llm.ResponseSchema{}, err
}
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
Key: ResponseSchemaKey,
ID: ResponseSchemaID,
Version: SchemaVersion,
Name: ResponseSchemaName,
AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json",
})
}

View File

@@ -2,9 +2,10 @@ package combat_semantics
import (
"context"
"fmt"
"encoding/json"
"strings"
"unicode/utf8"
"go.maximumdirect.net/promptkit"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -14,44 +15,20 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
)
const (
Key = "extract/dnd/scene-descriptions/combat_semantics"
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
maximumExplanationRunes = 512
)
const Key = "extract/dnd/scene-descriptions/combat_semantics"
const PromptID = "notarius/dnd.scene_descriptions.validate_combat"
type Options struct{}
type completionResponse struct {
Classification string `json:"classification"`
Explanation string `json:"explanation"`
}
type Validator struct {
llm contracts.StructuredLLMClient
promptSHA string
responseSchemaSHA string
}
type Validator struct{ engine *promptkit.Engine }
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
if llmClient == nil {
return nil, validatorErrorf("LLM client must not be nil")
func New(engine *promptkit.Engine, _ Options) (*Validator, error) {
if engine == nil {
return nil, validatorErrorf("native engine must not be nil")
}
promptSHA, err := promptAssetMetadata()
if err != nil {
return nil, validatorErrorf("load prompt metadata: %w", err)
}
responseSchema, err := loadResponseSchema()
if err != nil {
return nil, validatorErrorf("load response schema: %w", err)
}
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
return &Validator{engine: engine}, nil
}
func (v *Validator) Name() string { return Key }
@@ -60,62 +37,46 @@ func (v *Validator) ExecutionClass() contracts.ExecutionClass {
return contracts.ExecutionClassLLMBacked
}
func (v *Validator) ManifestMetadata() map[string]any {
if v == nil {
return nil
}
return map[string]any{
"prompt_id": PromptID,
"prompt_version": SchemaVersion,
"prompt_sha256": v.promptSHA,
"response_schema_key": string(ResponseSchemaKey),
"response_schema_id": ResponseSchemaID,
"response_schema_name": ResponseSchemaName,
"response_schema_version": SchemaVersion,
"response_schema_sha256": v.responseSchemaSHA,
"combat_policy": combatPolicy,
}
}
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
if v == nil {
return nil
}
return []pipeline.CheckpointFingerprint{
{Name: "prompt", Value: v.promptSHA},
{Name: "response_schema", Value: v.responseSchemaSHA},
{Name: "combat_policy", Value: combatPolicy},
}
}
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
if v == nil {
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
}
if v.llm == nil {
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
if v.engine == nil {
return contracts.ValidationResult{}, validatorErrorf("native engine must not be nil")
}
sourceInput, scene, err := validateRequest(ctx, req)
if err != nil {
return contracts.ValidationResult{}, err
}
var response completionResponse
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
StageName: Key,
PromptID: PromptID,
PromptVersion: SchemaVersion,
ProfileID: req.LLMProfile,
SessionID: req.SessionID,
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
Inputs: contracts.LLMInputSet{
"transcript": shared.TranscriptPromptMaterial(sourceInput),
},
}, &response)
if err != nil {
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
material := shared.TranscriptPromptMaterial(sourceInput)
result, err := v.engine.Run(ctx, promptkit.RunRequest{
PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.InlineWithURI(material.OriginURI, string(material.Content))},
Execution: req.Execution.Execution, TransientRetries: req.Execution.TransientRetries,
Observer: req.Execution.Observer, OnPrepared: req.Execution.OnPrepared,
})
if canceled := ctx.Err(); canceled != nil {
return contracts.ValidationResult{}, canceled
}
return interpretResponse(response, scene.Kind)
if err != nil {
return contracts.ValidationResult{}, unavailable(err)
}
if result == nil || result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return contracts.ValidationResult{}, unavailable(validatorErrorf("native output contract failed: %w", promptkit.ErrValidation))
}
var response classificationResponse
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
return contracts.ValidationResult{}, unavailable(validatorErrorf("decode native output: %w", err))
}
assessment, err := interpretResponse(response, scene.Kind)
if canceled := ctx.Err(); canceled != nil {
return contracts.ValidationResult{}, canceled
}
if err != nil {
return contracts.ValidationResult{}, unavailable(err)
}
return assessment, nil
}
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
@@ -128,7 +89,7 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
if req.Stage != string(pipeline.StageExtract) {
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
}
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
if err != nil {
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
}
@@ -154,53 +115,12 @@ func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[d
return sourceInput, scene, nil
}
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
explanation, err := validateExplanation(response.Explanation)
if err != nil {
return contracts.ValidationResult{}, err
}
switch response.Classification {
case "combat":
if proposedKind == dnd.SceneKindCombat {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeActiveCombatNotClassified,
Message: "The current chunk contains substantive active combat that is not classified as combat.",
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
}, nil
case "non_combat":
if proposedKind != dnd.SceneKindCombat {
return contracts.ValidationResult{Approved: true}, nil
}
return contracts.ValidationResult{
Approved: false,
ReasonCode: ReasonCodeCombatClassificationUnsupported,
Message: "The current chunk does not support a combat classification.",
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
}, nil
default:
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics classification %q", response.Classification)
}
}
func validateExplanation(value string) (string, error) {
if !utf8.ValidString(value) {
return "", validatorErrorf("validator explanation must be valid UTF-8")
}
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return "", validatorErrorf("validator explanation must not be blank")
}
if utf8.RuneCountInString(trimmed) > maximumExplanationRunes {
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
}
return trimmed, nil
func unavailable(cause error) error {
return &contracts.ValidationUnavailableError{Cause: validatorErrorf("native assessment unavailable: %w", cause)}
}
func Spec() pipeline.ValidatorSpec {
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID}}
}
func Register(registry *pipeline.ValidatorRegistry) error {
@@ -209,7 +129,7 @@ func Register(registry *pipeline.ValidatorRegistry) error {
if err != nil {
return nil, err
}
return New(request.Dependencies.LLM, options)
return New(request.Dependencies.Engine, options)
})
}
@@ -221,7 +141,3 @@ func DecodeOptions(options map[string]any) (Options, error) {
}
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
func validatorErrorf(format string, args ...any) error {
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
}

View File

@@ -4,75 +4,31 @@ import (
"context"
"encoding/json"
"errors"
"io"
"os"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/assets"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestValidatorInterpretsCombatSemanticsResponses(t *testing.T) {
request := validRequest(dnd.SceneKindNarrative)
for _, test := range []struct {
name string
kind dnd.SceneKind
response completionResponse
approved bool
reasonCode string
guidance string
correction string
wantError string
}{
{name: "approve non-combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}, approved: true},
{name: "approve combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "combat", Explanation: "Initiative and attacks organize the chunk."}, approved: true},
{name: "add combat", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: "The combatants exchange attacks and damage."}, reasonCode: ReasonCodeActiveCombatNotClassified, guidance: "Return kind: combat"},
{name: "remove combat", kind: dnd.SceneKindCombat, response: completionResponse{Classification: "non_combat", Explanation: "The group only plans for a possible fight."}, reasonCode: ReasonCodeCombatClassificationUnsupported, guidance: "narrative, recap, or meta"},
{name: "unknown classification", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "uncertain", Explanation: "The group only plans."}, wantError: "unsupported"},
{name: "blank explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: " "}, wantError: "blank"},
{name: "trim explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "combat", Explanation: " The combatants exchange attacks.\n"}, reasonCode: ReasonCodeActiveCombatNotClassified, correction: "Return kind: combat for this scene. The combatants exchange attacks."},
{name: "oversized explanation", kind: dnd.SceneKindNarrative, response: completionResponse{Classification: "non_combat", Explanation: strings.Repeat("界", 513)}, wantError: "exceeds"},
} {
t.Run(test.name, func(t *testing.T) {
request.Value.Scenes[0].Kind = test.kind
client := &fakeCombatSemanticsClient{response: test.response}
validator := newValidator(t, client)
result, err := validator.Validate(context.Background(), request)
if test.wantError != "" {
if err == nil || !strings.Contains(err.Error(), test.wantError) {
t.Fatalf("Validate() error = %v, want %q", err, test.wantError)
}
return
}
if err != nil || result.Approved != test.approved || result.ReasonCode != test.reasonCode || !strings.Contains(result.CorrectionGuidance, test.guidance) {
t.Fatalf("Validate() = %#v, %v", result, err)
}
if test.correction != "" && result.CorrectionGuidance != test.correction {
t.Fatalf("CorrectionGuidance = %q, want %q", result.CorrectionGuidance, test.correction)
}
if !result.Approved && (strings.Contains(result.CorrectionGuidance, Key) || strings.Contains(result.CorrectionGuidance, result.ReasonCode) || strings.Contains(result.CorrectionGuidance, request.Chunk.ID)) {
t.Fatalf("correction guidance leaked opaque application detail: %q", result.CorrectionGuidance)
}
if err := contracts.ValidateValidationResult(result); err != nil {
t.Fatalf("ValidateValidationResult(%#v) = %v", result, err)
}
})
}
}
func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
request := validRequest(dnd.SceneKindNarrative)
repairAttempts := 2
request.LLMProfile = "validator-profile"
request.SessionID = "validator-session"
request.StructuredOutputRepairAttempts = &repairAttempts
request.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{"players": {Items: []contracts.ReferenceItem{{Content: []byte("hidden reference")}}}}}
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
validator := newValidator(t, client)
model := "validator-override"
request.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: model}
var prepared promptkit.PreparedRun
preparedCalls := 0
request.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = value; preparedCalls++ }
if result, err := validator.Validate(context.Background(), request); err != nil || !result.Approved {
t.Fatalf("Validate() = %#v, %v", result, err)
}
@@ -80,19 +36,22 @@ func TestValidatorUsesOnlyChunkTranscript(t *testing.T) {
t.Fatalf("completion count = %d, want 1", len(client.requests))
}
completed := client.requests[0]
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != request.LLMProfile || completed.SessionID != request.SessionID || completed.StructuredOutputRepairAttempts == request.StructuredOutputRepairAttempts || *completed.StructuredOutputRepairAttempts != repairAttempts {
t.Fatalf("completion request = %#v", completed)
if preparedCalls != 1 || prepared.PromptID != PromptID || prepared.SelectedProfileID != request.LLMProfile || completed.Prompt.SessionID != request.SessionID || completed.Target.Model != model {
t.Fatal("native execution selection/override missing")
}
if completed.Correction != nil || !reflect.DeepEqual(sortedInputNames(completed.Inputs), []string{"transcript"}) || !reflect.DeepEqual(completed.Inputs["transcript"].Content, request.Chunk.Content) {
t.Fatalf("completion inputs = %#v", completed.Inputs)
var messages string
for _, message := range completed.Prompt.Messages {
messages += message.Content
}
if strings.Count(messages, string(request.Chunk.Content)) != 1 {
t.Fatal("chunk transcript not rendered exactly once")
}
for _, forbidden := range []string{request.Value.Scenes[0].ID, request.Value.Scenes[0].Title, request.Value.Scenes[0].Summary, "hidden reference"} {
for name, input := range completed.Inputs {
if strings.Contains(string(input.Content), forbidden) {
t.Fatalf("input %q leaked forbidden model-visible value %q", name, forbidden)
}
if strings.Contains(messages, forbidden) {
t.Fatalf("model-visible context leaked %q", forbidden)
}
}
}
func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
@@ -130,7 +89,7 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
if test.mutate != nil {
test.mutate(&request)
}
client := &fakeCombatSemanticsClient{response: completionResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
client := &fakeCombatSemanticsClient{response: classificationResponse{Classification: "non_combat", Explanation: "No active encounter occurs."}}
_, err := newValidator(t, client).Validate(test.ctx, request)
if err == nil || !strings.Contains(err.Error(), test.wantErr) || len(client.requests) != 0 {
t.Fatalf("Validate() error = %v, calls = %d; want %q and no calls", err, len(client.requests), test.wantErr)
@@ -138,29 +97,26 @@ func TestValidatorRejectsInvalidRequestsAndCompletionFailures(t *testing.T) {
})
}
client := &fakeCombatSemanticsClient{err: errors.New("provider unavailable")}
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "complete structured output") {
if _, err := newValidator(t, client).Validate(context.Background(), base); err == nil || !strings.Contains(err.Error(), "assessment unavailable") {
t.Fatalf("completion failure error = %v", err)
}
}
func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
func TestValidatorConstructionOptionsAndDependencies(t *testing.T) {
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
t.Fatalf("New(nil) error = %v", err)
}
if _, err := DecodeOptions(map[string]any{"unexpected": true}); err == nil {
t.Fatal("DecodeOptions() accepted unknown options")
}
validator := newValidator(t, &fakeCombatSemanticsClient{})
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || Spec().CorrectionProtocol != "" {
if validator.Name() != Key || validator.ExecutionClass() != contracts.ExecutionClassLLMBacked || !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
t.Fatalf("validator contract = %#v / %#v", validator, Spec())
}
metadata, err := json.Marshal(validator.ManifestMetadata())
if err != nil || !strings.Contains(string(metadata), ResponseSchemaID) || strings.Contains(string(metadata), "non_combat") {
t.Fatalf("manifest metadata = %s, %v", metadata, err)
}
fingerprints := validator.CheckpointFingerprints()
if len(fingerprints) != 3 || fingerprints[0].Name != "prompt" || fingerprints[1].Name != "response_schema" || fingerprints[2] != (pipeline.CheckpointFingerprint{Name: "combat_policy", Value: combatPolicy}) {
t.Fatalf("fingerprints = %#v", fingerprints)
selected := Spec()
selected.Prompts[0] = "mutated"
if !reflect.DeepEqual(Spec().Prompts, []string{PromptID}) {
t.Fatal("prompt dependencies alias caller memory")
}
registry := pipeline.NewValidatorRegistry()
if err := Register(registry); err != nil {
@@ -171,60 +127,36 @@ func TestValidatorConstructionOptionsAndMetadata(t *testing.T) {
}
}
func TestEvaluationCasesAreStrictAndCoverClassifications(t *testing.T) {
content, err := os.ReadFile("testdata/evaluation_cases.json")
func newValidator(t *testing.T, client promptkit.LLMClient) *Validator {
t.Helper()
type settings struct {
PromptKit promptkit.Settings `yaml:"promptkit"`
}
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("validator fixture", []byte(`promptkit:
backends:
- id: validator-backend
endpoint: https://example.invalid/v1
concurrency_limit: 1
queue_capacity: 0
profiles:
- id: validator-profile
backend: validator-backend
model: validator-test-model
api_key_required: false
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
promptkit.BundledOpenRouterUnderlay(),
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
}})
if err != nil {
t.Fatal(err)
}
var cases []struct {
Name string `json:"name"`
TranscriptUnits []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"transcript_units"`
ProposedKind string `json:"proposed_kind"`
ExpectedClassification string `json:"expected_classification"`
ReviewerRationale string `json:"reviewer_rationale"`
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
decoder := json.NewDecoder(strings.NewReader(string(content)))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&cases); err != nil {
t.Fatalf("decode evaluation cases: %v", err)
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
t.Fatalf("decode trailing evaluation data: %v, want EOF", err)
}
seenNames, seenClassifications := map[string]bool{}, map[string]bool{}
for _, value := range cases {
if strings.TrimSpace(value.Name) == "" || seenNames[value.Name] || len(value.TranscriptUnits) == 0 || strings.TrimSpace(value.ProposedKind) == "" || strings.TrimSpace(value.ReviewerRationale) == "" {
t.Fatalf("invalid evaluation case: %#v", value)
}
seenNames[value.Name] = true
switch value.ProposedKind {
case "combat", "narrative", "recap", "meta":
default:
t.Fatalf("unsupported proposed kind %q", value.ProposedKind)
}
switch value.ExpectedClassification {
case "combat", "non_combat":
seenClassifications[value.ExpectedClassification] = true
default:
t.Fatalf("unsupported classification %q", value.ExpectedClassification)
}
for _, unit := range value.TranscriptUnits {
if unit.ID <= 0 || strings.TrimSpace(unit.Text) == "" {
t.Fatalf("invalid transcript unit: %#v", unit)
}
}
}
if len(seenClassifications) != 2 {
t.Fatalf("evaluation classifications = %#v, want both classes", seenClassifications)
}
}
func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator {
t.Helper()
validator, err := New(client, Options{})
validator, err := New(engine, Options{})
if err != nil {
t.Fatalf("New() error = %v", err)
}
@@ -234,7 +166,7 @@ func newValidator(t *testing.T, client contracts.StructuredLLMClient) *Validator
func validRequest(kind dnd.SceneKind) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
document := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1, Text: "The foes attack."}, {ID: 2, Text: "The party responds."}}}
chunk := &source.Chunk{ID: "session:chunk:0", SourceID: document.ID, Ref: source.SourceRef{SourceID: document.ID, StartUnitID: 1, EndUnitID: 2}, Content: []byte(`{"units":[1,2]}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), document.Units...)}
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
return contracts.TypedValidationRequest[dnd.SceneDescriptionList]{Stage: string(pipeline.StageExtract), LLMProfile: "validator-profile", Source: document, Chunk: chunk, SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session.json"), Value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: chunk.ID, SourceRef: chunk.Ref, Kind: kind, Title: "A Scene", Summary: "A valid scene."}}}}
}
func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDescriptionList]) contracts.TypedValidationRequest[dnd.SceneDescriptionList] {
@@ -247,40 +179,28 @@ func cloneValidationRequest(request contracts.TypedValidationRequest[dnd.SceneDe
return request
}
func sortedInputNames(inputs contracts.LLMInputSet) []string {
names := make([]string, 0, len(inputs))
for name := range inputs {
names = append(names, name)
}
if len(names) == 2 && names[0] == "transcript" {
names[0], names[1] = names[1], names[0]
}
return names
}
type fakeCombatSemanticsClient struct {
response completionResponse
err error
requests []contracts.StructuredCompletionRequest
response classificationResponse
raw string
err error
requests []promptkit.GenerateRequest
onGenerate func(context.Context, promptkit.GenerateRequest)
}
func (client *fakeCombatSemanticsClient) CompleteStructured(_ context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
cloned, err := contracts.CloneStructuredCompletionRequest(request)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
func (client *fakeCombatSemanticsClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.requests = append(client.requests, request)
if client.onGenerate != nil {
client.onGenerate(ctx, request)
}
client.requests = append(client.requests, cloned)
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
return nil, client.err
}
response, ok := out.(*completionResponse)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected completion target")
if client.raw != "" {
return &promptkit.GenerateResponse{Content: client.raw}, nil
}
*response = client.response
content, err := json.Marshal(client.response)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return nil, err
}
return contracts.StructuredCompletionResponse{Content: content}, nil
return &promptkit.GenerateResponse{Content: string(content)}, nil
}