Verify native recovery, fallback reuse boundaries, and operator schemas

This commit is contained in:
2026-11-22 06:00:52 +00:00
parent 854d23b91e
commit 5b6854e975
3 changed files with 369 additions and 4 deletions

View File

@@ -1780,6 +1780,36 @@ them; live OpenRouter/local services are not default validation requirements.
## Stage 37: Verify Assembled Recovery And Native Command Workflows
**Status:** Complete. Added two focused native CLI recovery contracts using the
maintained ordered D&D composition, existing provider fake, real domain modules,
typed generated references, filesystem cache, and capture. They include semantic
normalization and the real combat-semantics LLM validator on one limited backend
with one shared worker. A final-turn corrected scene is stored under the initial
prepared message hash; a downstream failure preserves accepted predecessors and
the chunk plan, and fixed reruns execute only the previously failed work. Full
hits retain capture without native executions or historical accounting. Native
failure causes, session/model/backend selections, unknown usage, terminal group
coverage, domain payloads, and source identity remain intact.
A failed permissive NPC refresh emits one qualified fallback, forces actual
generated-reference consumers to execute without reuse, leaves unrelated lanes
cached, and preserves earlier accepted files. Restoring ordinary mode reuses those
files without provider work. Operator-schema tests now execute the same prepared
handle and verify the custom provider constraint and native validation for inline,
source-relative, and external-root schemas; missing replacement roots do not fall
back to embedded schemas. No compatibility/configuration translation was added.
Reviewed and retained the existing focused producer/assessment/cache/capture,
worker/admission, command/preflight, and report/subprocess coverage for structural
failure with nil error, result-less/undelivered responses, full ordered guidance,
final-turn assessment, unchanged candidates, native overrides, cancellation,
stream/delivery outcomes, and no duplicate debug preparation. Explicit test repair
budgets use complete native output contracts, without pinning upstream defaults.
All four prescribed package suites and vet pass, as do repeated focused CLI and
shared observer/coordinator/backend race checks and `git diff --check`. No real
credentials, external inference, or model-quality claim is involved. Stage 46
still owns final whole-roadmap verification after documentation updates.
**Goal:** prove the cross-boundary behavior most likely to regress in this cutover.
**Inspect/change:** CLI assembled/command/cache contracts, module integration

View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"path/filepath"
"testing"
@@ -14,7 +15,7 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
name, schema, root string
missing bool
}{
{"inline", "schema: {type: object}", "", false},
{"inline", `schema: {type: object, required: [operator_value], properties: {operator_value: {const: selected}}, additionalProperties: false}`, "", false},
{"source file", "schema_file: response.json", "", false},
{"external root", "schema_path: response.json", "schema_root: external-schemas", false},
{"replaced root missing embedded schema", "schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json", "schema_root: external-schemas", true},
@@ -23,10 +24,18 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
writeConfigKitFile(t, main, "version: 6\n")
writeConfigKitFile(t, filepath.Join(dir, "response.json"), `{"type":"object"}`)
writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), `{"type":"object"}`)
schema := `{"type":"object","required":["operator_value"],"properties":{"operator_value":{"const":"selected"}},"additionalProperties":false}`
writeConfigKitFile(t, filepath.Join(dir, "response.json"), schema)
writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), schema)
writeConfigKitFile(t, filepath.Join(dir, "prompts.yml"), "collections:\n notarius:\n "+test.root+"\n prompts:\n - id: dnd.scenes\n default_profile: notarius/dnd-extraction\n messages: [{role: user, content: operator-message}]\n output:\n format: json\n validation_mode: json_schema\n "+test.schema+"\n")
loaders := &nativeLoaders{}
calls := 0
loaders := &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
calls++
if !productionSchemaHasProperty(request, "operator_value") {
return nil, errors.New("operator schema did not reach provider preparation")
}
return &promptkit.GenerateResponse{Content: `{"operator_value":"selected"}`}, nil
}))}}
engine, err := loaders.engine(context.Background(), nativeSelection(main))
if err != nil {
t.Fatal(err)
@@ -48,6 +57,10 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
if len(details.Messages) != 1 || details.Messages[0].Content != "operator-message" || details.StructuredOutput == nil {
t.Fatalf("replacement contract = %#v", details)
}
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || !result.Validation.IsValid || calls != 1 {
t.Fatalf("operator contract execution=%#v error=%v calls=%d", result, err, calls)
}
// Replacing one identified prompt must not discard unrelated embedded prompts.
if _, err := engine.InspectPrompt(context.Background(), "notarius/dnd.spells"); err != nil {
t.Fatal(err)

View File

@@ -0,0 +1,322 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
// Use the maintained ordered domain composition and existing provider fake.
// Only external failures and one rejected scene candidate are controlled here.
type recoveryClient struct {
base enemyEventLLMClient
mu sync.Mutex
requests []promptkit.GenerateRequest
failEvents, failNPC, correctScene bool
failure error
}
func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
role := enemyEventRequestRole(request)
if productionSchemaHasProperty(request, "classification") {
role = "classification"
}
client.mu.Lock()
client.requests = append(client.requests, request)
failEvents, failNPC := client.failEvents, client.failNPC
transcript, _ := productionJSONInput(request, "units")
combat := strings.Contains(string(transcript), "Roll initiative")
incorrectScene := role == "kind" && combat && client.correctScene
if incorrectScene {
client.correctScene = false
}
client.mu.Unlock()
if role == "events" && failEvents || role == "npcs" && combat && failNPC {
return nil, client.failure
}
if incorrectScene {
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil
}
if role == "classification" {
classification := "non_combat"
if combat {
classification = "combat"
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil
}
// Empty generated registries remain valid current-run context. Do not invent
// identities or events merely to fill the dependent lanes in the fallback run.
if role == "npc-occurrences" || role == "combat_turns" || role == "events" {
if content, err := productionJSONInput(request, "npcs"); err == nil {
var registry struct {
NPCs []json.RawMessage `json:"npcs"`
}
if err := json.Unmarshal(content, &registry); err != nil {
return nil, err
}
if len(registry.NPCs) == 0 {
property := "occurrences"
if role == "combat_turns" {
property = "combat_turns"
}
if role == "events" {
property = "events"
}
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil
}
}
}
return client.base.Generate(ctx, request)
}
func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) {
client.mu.Lock()
defer client.mu.Unlock()
counts := map[string]int{}
for _, request := range client.requests {
role := enemyEventRequestRole(request)
if productionSchemaHasProperty(request, "classification") {
role = "classification"
}
counts[role]++
}
return counts, append([]promptkit.GenerateRequest(nil), client.requests...)
}
type recoveryFixture struct {
config, cache, output, debug string
client *recoveryClient
options Options
}
func newRecoveryFixture(t *testing.T) *recoveryFixture {
t.Helper()
root := t.TempDir()
for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} {
content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name))
if err != nil {
t.Fatal(err)
}
writeConfigKitFile(t, filepath.Join(root, name), string(content))
}
description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml")
content, err := os.ReadFile(description)
if err != nil {
t.Fatal(err)
}
configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract:
module: dnd/scene-descriptions
validators:
- module: generic/valid_json
- module: extract/dnd/scene-descriptions/shape
- module: extract/dnd/scene-descriptions/source_refs
- module: generic/valid_json_schema
- module: extract/dnd/scene-descriptions/source_relatedness
- module: extract/dnd/scene-descriptions/combat_semantics
`)
writeConfigKitFile(t, description, configured)
fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}}
writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6
config:
imports:
- path: pipelines/dnd-session.yml
required: true
output: {directory: %q}
cache_directory: %q
debug: {directory: %q}
concurrency: {workers: 1}
promptkit:
backends:
- id: fixture
endpoint: https://example.invalid/v1
concurrency_limit: 1
collections:
notarius:
profiles:
- id: dnd-extraction
backend: fixture
model: recovery-model
api_key_required: false
timeout_seconds: 2
- id: dnd-extraction-example
extends: {id: dnd-extraction}
`, fixture.output, fixture.cache, fixture.debug))
fixture.options = productionRunOptions(t, fixture.client)
run := 0
fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil }
return fixture
}
func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) {
t.Helper()
args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0",
"--set", "/pipelines/dnd-session/max_additional_turns=1",
}
app, err := NewApplication("development", fixture.options)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var stdout, stderr bytes.Buffer
outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr})
var report runResult
if outcome.Status == 0 {
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
t.Fatalf("completion=%q: %v", stdout.String(), err)
}
} else if stdout.Len() != 0 {
t.Fatalf("failed operation emitted success output: %q", stdout.String())
}
return outcome, report, stderr.String()
}
func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) {
fixture := newRecoveryFixture(t)
fixture.client.failEvents, fixture.client.correctScene = true, true
outcome, _, diagnostics := fixture.run(t)
if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) {
t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics)
}
assertAbsent(t, fixture.output)
firstDebug := filepath.Join(fixture.debug, "recovery-1")
assertFile(t, filepath.Join(firstDebug, "outcome.json"))
counts, requests := fixture.client.snapshot()
if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 {
t.Fatalf("ordered native roles/correction=%v", counts)
}
for _, request := range requests {
if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" {
t.Fatalf("lost shared native selection: %#v", request)
}
}
summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json"))
if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known {
t.Fatalf("failed-run native facts lost or duplicated: %#v", summary)
}
// Locate only the initial scene request's native hash in available capture;
// assert domain bytes at that key, not a persisted execution ledger.
var correctedKey string
for name, content := range readTree(t, firstDebug) {
if !strings.HasSuffix(name, "/prepared.json") {
continue
}
var details promptkit.PreparedRun
if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" {
continue
}
for _, message := range details.Messages {
if strings.Contains(message.Content, "Roll initiative") {
correctedKey = details.MessageContentHash
}
}
}
if correctedKey == "" {
t.Fatal("no prepared initial combat scene evidence")
}
corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json"))
if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat {
t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected)
}
fixture.client.failEvents = false
outcome, report, diagnostics := fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 {
t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
after, _ := fixture.client.snapshot()
for role, count := range counts {
want := count
if role == "events" {
want++
}
if after[role] != want {
t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after)
}
}
artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json"))
if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" {
t.Fatalf("recovered domain evidence=%#v", artifact)
}
stable := readTree(t, fixture.cache)
outcome, hitReport, diagnostics := fixture.run(t)
if outcome.Status != 0 || hitReport.ArtifactCount != 12 {
t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics)
}
last, _ := fixture.client.snapshot()
if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) {
t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last)
}
for name := range readTree(t, hitReport.DebugDirectory) {
if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") {
t.Fatalf("full hit fabricated native execution: %s", name)
}
}
hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json"))
if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete {
t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary)
}
}
func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) {
fixture := newRecoveryFixture(t)
outcome, _, diagnostics := fixture.run(t)
if outcome.Status != 0 {
t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics)
}
before, _ := fixture.client.snapshot()
accepted := readTree(t, fixture.cache)
fixture.client.failNPC = true
description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml")
content, err := os.ReadFile(description)
if err != nil {
t.Fatal(err)
}
writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n"))
outcome, report, diagnostics := fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 1 {
t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
after, _ := fixture.client.snapshot()
for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} {
if after[role] != before[role] {
t.Fatalf("unaffected %s reran: %v -> %v", role, before, after)
}
}
for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} {
if after[role] <= before[role] {
t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after)
}
}
if !sameFiles(accepted, readTree(t, fixture.cache)) {
t.Fatal("failed refresh/fallback replaced accepted cache data")
}
registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json"))
if registry.NPCs == nil || len(registry.NPCs) != 0 {
t.Fatalf("fallback registry=%#v", registry)
}
fixture.client.failNPC = false
writeConfigKitFile(t, description, string(content))
outcome, report, diagnostics = fixture.run(t)
if outcome.Status != 0 || report.FallbackCount != 0 {
t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
}
last, _ := fixture.client.snapshot()
if !reflect.DeepEqual(last, after) {
t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last)
}
}
var _ promptkit.LLMClient = (*recoveryClient)(nil)