Verify native recovery, fallback reuse boundaries, and operator schemas
This commit is contained in:
@@ -1780,6 +1780,36 @@ them; live OpenRouter/local services are not default validation requirements.
|
||||
|
||||
## Stage 37: Verify Assembled Recovery And Native Command Workflows
|
||||
|
||||
**Status:** Complete. Added two focused native CLI recovery contracts using the
|
||||
maintained ordered D&D composition, existing provider fake, real domain modules,
|
||||
typed generated references, filesystem cache, and capture. They include semantic
|
||||
normalization and the real combat-semantics LLM validator on one limited backend
|
||||
with one shared worker. A final-turn corrected scene is stored under the initial
|
||||
prepared message hash; a downstream failure preserves accepted predecessors and
|
||||
the chunk plan, and fixed reruns execute only the previously failed work. Full
|
||||
hits retain capture without native executions or historical accounting. Native
|
||||
failure causes, session/model/backend selections, unknown usage, terminal group
|
||||
coverage, domain payloads, and source identity remain intact.
|
||||
|
||||
A failed permissive NPC refresh emits one qualified fallback, forces actual
|
||||
generated-reference consumers to execute without reuse, leaves unrelated lanes
|
||||
cached, and preserves earlier accepted files. Restoring ordinary mode reuses those
|
||||
files without provider work. Operator-schema tests now execute the same prepared
|
||||
handle and verify the custom provider constraint and native validation for inline,
|
||||
source-relative, and external-root schemas; missing replacement roots do not fall
|
||||
back to embedded schemas. No compatibility/configuration translation was added.
|
||||
|
||||
Reviewed and retained the existing focused producer/assessment/cache/capture,
|
||||
worker/admission, command/preflight, and report/subprocess coverage for structural
|
||||
failure with nil error, result-less/undelivered responses, full ordered guidance,
|
||||
final-turn assessment, unchanged candidates, native overrides, cancellation,
|
||||
stream/delivery outcomes, and no duplicate debug preparation. Explicit test repair
|
||||
budgets use complete native output contracts, without pinning upstream defaults.
|
||||
All four prescribed package suites and vet pass, as do repeated focused CLI and
|
||||
shared observer/coordinator/backend race checks and `git diff --check`. No real
|
||||
credentials, external inference, or model-quality claim is involved. Stage 46
|
||||
still owns final whole-roadmap verification after documentation updates.
|
||||
|
||||
**Goal:** prove the cross-boundary behavior most likely to regress in this cutover.
|
||||
|
||||
**Inspect/change:** CLI assembled/command/cache contracts, module integration
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
@@ -14,7 +15,7 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
|
||||
name, schema, root string
|
||||
missing bool
|
||||
}{
|
||||
{"inline", "schema: {type: object}", "", false},
|
||||
{"inline", `schema: {type: object, required: [operator_value], properties: {operator_value: {const: selected}}, additionalProperties: false}`, "", false},
|
||||
{"source file", "schema_file: response.json", "", false},
|
||||
{"external root", "schema_path: response.json", "schema_root: external-schemas", false},
|
||||
{"replaced root missing embedded schema", "schema_path: dnd/scenes/schemas/dnd_scenes_llm.v1.json", "schema_root: external-schemas", true},
|
||||
@@ -23,10 +24,18 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
writeConfigKitFile(t, main, "version: 6\n")
|
||||
writeConfigKitFile(t, filepath.Join(dir, "response.json"), `{"type":"object"}`)
|
||||
writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), `{"type":"object"}`)
|
||||
schema := `{"type":"object","required":["operator_value"],"properties":{"operator_value":{"const":"selected"}},"additionalProperties":false}`
|
||||
writeConfigKitFile(t, filepath.Join(dir, "response.json"), schema)
|
||||
writeConfigKitFile(t, filepath.Join(dir, "external-schemas", "response.json"), schema)
|
||||
writeConfigKitFile(t, filepath.Join(dir, "prompts.yml"), "collections:\n notarius:\n "+test.root+"\n prompts:\n - id: dnd.scenes\n default_profile: notarius/dnd-extraction\n messages: [{role: user, content: operator-message}]\n output:\n format: json\n validation_mode: json_schema\n "+test.schema+"\n")
|
||||
loaders := &nativeLoaders{}
|
||||
calls := 0
|
||||
loaders := &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(captureClient(func(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
calls++
|
||||
if !productionSchemaHasProperty(request, "operator_value") {
|
||||
return nil, errors.New("operator schema did not reach provider preparation")
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: `{"operator_value":"selected"}`}, nil
|
||||
}))}}
|
||||
engine, err := loaders.engine(context.Background(), nativeSelection(main))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -48,6 +57,10 @@ func TestNativeLoadPreparesOperatorSchemaSelections(t *testing.T) {
|
||||
if len(details.Messages) != 1 || details.Messages[0].Content != "operator-message" || details.StructuredOutput == nil {
|
||||
t.Fatalf("replacement contract = %#v", details)
|
||||
}
|
||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||
if err != nil || result == nil || !result.Validation.IsValid || calls != 1 {
|
||||
t.Fatalf("operator contract execution=%#v error=%v calls=%d", result, err, calls)
|
||||
}
|
||||
// Replacing one identified prompt must not discard unrelated embedded prompts.
|
||||
if _, err := engine.InspectPrompt(context.Background(), "notarius/dnd.spells"); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
322
internal/cli/recovery_contract_test.go
Normal file
322
internal/cli/recovery_contract_test.go
Normal file
@@ -0,0 +1,322 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// Use the maintained ordered domain composition and existing provider fake.
|
||||
// Only external failures and one rejected scene candidate are controlled here.
|
||||
type recoveryClient struct {
|
||||
base enemyEventLLMClient
|
||||
mu sync.Mutex
|
||||
requests []promptkit.GenerateRequest
|
||||
failEvents, failNPC, correctScene bool
|
||||
failure error
|
||||
}
|
||||
|
||||
func (client *recoveryClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
role := enemyEventRequestRole(request)
|
||||
if productionSchemaHasProperty(request, "classification") {
|
||||
role = "classification"
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, request)
|
||||
failEvents, failNPC := client.failEvents, client.failNPC
|
||||
transcript, _ := productionJSONInput(request, "units")
|
||||
combat := strings.Contains(string(transcript), "Roll initiative")
|
||||
incorrectScene := role == "kind" && combat && client.correctScene
|
||||
if incorrectScene {
|
||||
client.correctScene = false
|
||||
}
|
||||
client.mu.Unlock()
|
||||
if role == "events" && failEvents || role == "npcs" && combat && failNPC {
|
||||
return nil, client.failure
|
||||
}
|
||||
if incorrectScene {
|
||||
return &promptkit.GenerateResponse{Content: `{"kind":"narrative","title":"Arrival","summary":"An initially missed combat scene."}`}, nil
|
||||
}
|
||||
if role == "classification" {
|
||||
classification := "non_combat"
|
||||
if combat {
|
||||
classification = "combat"
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"classification":%q,"explanation":"Classification follows the supplied scene transcript."}`, classification)}, nil
|
||||
}
|
||||
// Empty generated registries remain valid current-run context. Do not invent
|
||||
// identities or events merely to fill the dependent lanes in the fallback run.
|
||||
if role == "npc-occurrences" || role == "combat_turns" || role == "events" {
|
||||
if content, err := productionJSONInput(request, "npcs"); err == nil {
|
||||
var registry struct {
|
||||
NPCs []json.RawMessage `json:"npcs"`
|
||||
}
|
||||
if err := json.Unmarshal(content, ®istry); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(registry.NPCs) == 0 {
|
||||
property := "occurrences"
|
||||
if role == "combat_turns" {
|
||||
property = "combat_turns"
|
||||
}
|
||||
if role == "events" {
|
||||
property = "events"
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{%q:[]}`, property)}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return client.base.Generate(ctx, request)
|
||||
}
|
||||
|
||||
func (client *recoveryClient) snapshot() (map[string]int, []promptkit.GenerateRequest) {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
counts := map[string]int{}
|
||||
for _, request := range client.requests {
|
||||
role := enemyEventRequestRole(request)
|
||||
if productionSchemaHasProperty(request, "classification") {
|
||||
role = "classification"
|
||||
}
|
||||
counts[role]++
|
||||
}
|
||||
return counts, append([]promptkit.GenerateRequest(nil), client.requests...)
|
||||
}
|
||||
|
||||
type recoveryFixture struct {
|
||||
config, cache, output, debug string
|
||||
client *recoveryClient
|
||||
options Options
|
||||
}
|
||||
|
||||
func newRecoveryFixture(t *testing.T) *recoveryFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{"prompts.yml", "transcript.json", "party.txt", "glossary.txt", "spell-catalog.json", "pipelines/dnd-session.yml", "pipelines/dnd-session.describe-session.yml"} {
|
||||
content, err := os.ReadFile(repositoryPath("examples", "dnd-complete", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeConfigKitFile(t, filepath.Join(root, name), string(content))
|
||||
}
|
||||
description := filepath.Join(root, "pipelines", "dnd-session.describe-session.yml")
|
||||
content, err := os.ReadFile(description)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configured := replaceRequiredOnce(t, string(content), " extract:\n module: dnd/scene-descriptions\n", ` extract:
|
||||
module: dnd/scene-descriptions
|
||||
validators:
|
||||
- module: generic/valid_json
|
||||
- module: extract/dnd/scene-descriptions/shape
|
||||
- module: extract/dnd/scene-descriptions/source_refs
|
||||
- module: generic/valid_json_schema
|
||||
- module: extract/dnd/scene-descriptions/source_relatedness
|
||||
- module: extract/dnd/scene-descriptions/combat_semantics
|
||||
`)
|
||||
writeConfigKitFile(t, description, configured)
|
||||
fixture := &recoveryFixture{config: filepath.Join(root, "config.yml"), cache: filepath.Join(root, "cache"), output: filepath.Join(root, "output"), debug: filepath.Join(root, "debug"), client: &recoveryClient{failure: errors.New("controlled downstream provider failure")}}
|
||||
writeConfigKitFile(t, fixture.config, fmt.Sprintf(`version: 6
|
||||
config:
|
||||
imports:
|
||||
- path: pipelines/dnd-session.yml
|
||||
required: true
|
||||
output: {directory: %q}
|
||||
cache_directory: %q
|
||||
debug: {directory: %q}
|
||||
concurrency: {workers: 1}
|
||||
promptkit:
|
||||
backends:
|
||||
- id: fixture
|
||||
endpoint: https://example.invalid/v1
|
||||
concurrency_limit: 1
|
||||
collections:
|
||||
notarius:
|
||||
profiles:
|
||||
- id: dnd-extraction
|
||||
backend: fixture
|
||||
model: recovery-model
|
||||
api_key_required: false
|
||||
timeout_seconds: 2
|
||||
- id: dnd-extraction-example
|
||||
extends: {id: dnd-extraction}
|
||||
`, fixture.output, fixture.cache, fixture.debug))
|
||||
fixture.options = productionRunOptions(t, fixture.client)
|
||||
run := 0
|
||||
fixture.options.RunIDGenerator = func(time.Time) (string, error) { run++; return fmt.Sprintf("recovery-%d", run), nil }
|
||||
return fixture
|
||||
}
|
||||
|
||||
func (fixture *recoveryFixture) run(t *testing.T) (commandkit.Outcome, runResult, string) {
|
||||
t.Helper()
|
||||
args := []string{"run", "dnd-session", "--config", fixture.config, "--input-file", filepath.Join(filepath.Dir(fixture.config), "transcript.json"), "--debug", "--progress=false", "--format=json", "--llm-session-id=recovery-session", "--llm-transient-retries=0",
|
||||
"--set", "/pipelines/dnd-session/max_additional_turns=1",
|
||||
}
|
||||
app, err := NewApplication("development", fixture.options)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var stdout, stderr bytes.Buffer
|
||||
outcome := app.Execute(ctx, args, commandkit.Streams{Out: &stdout, Err: &stderr})
|
||||
var report runResult
|
||||
if outcome.Status == 0 {
|
||||
if err := json.Unmarshal(stdout.Bytes(), &report); err != nil {
|
||||
t.Fatalf("completion=%q: %v", stdout.String(), err)
|
||||
}
|
||||
} else if stdout.Len() != 0 {
|
||||
t.Fatalf("failed operation emitted success output: %q", stdout.String())
|
||||
}
|
||||
return outcome, report, stderr.String()
|
||||
}
|
||||
|
||||
func TestAssembledRecoveryReusesCorrectedPredecessorsAndChunkPlan(t *testing.T) {
|
||||
fixture := newRecoveryFixture(t)
|
||||
fixture.client.failEvents, fixture.client.correctScene = true, true
|
||||
outcome, _, diagnostics := fixture.run(t)
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, fixture.client.failure) {
|
||||
t.Fatalf("downstream failure=%#v diagnostics=%q", outcome, diagnostics)
|
||||
}
|
||||
assertAbsent(t, fixture.output)
|
||||
firstDebug := filepath.Join(fixture.debug, "recovery-1")
|
||||
assertFile(t, filepath.Join(firstDebug, "outcome.json"))
|
||||
counts, requests := fixture.client.snapshot()
|
||||
if counts["scenes"] != 1 || counts["kind"] != 3 || counts["classification"] != 3 || counts["duplicate_groups"] == 0 || counts["events"] != 1 {
|
||||
t.Fatalf("ordered native roles/correction=%v", counts)
|
||||
}
|
||||
for _, request := range requests {
|
||||
if request.Prompt.SessionID != "recovery-session" || request.Target.Model != "recovery-model" || request.Target.BackendID != "fixture" {
|
||||
t.Fatalf("lost shared native selection: %#v", request)
|
||||
}
|
||||
}
|
||||
summary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(firstDebug, "executions.json"))
|
||||
if len(summary.Members) != len(requests) || summary.Attempts != uint64(len(requests)) || !summary.TerminalCoverageComplete || summary.Usage.TotalTokens.Known {
|
||||
t.Fatalf("failed-run native facts lost or duplicated: %#v", summary)
|
||||
}
|
||||
// Locate only the initial scene request's native hash in available capture;
|
||||
// assert domain bytes at that key, not a persisted execution ledger.
|
||||
var correctedKey string
|
||||
for name, content := range readTree(t, firstDebug) {
|
||||
if !strings.HasSuffix(name, "/prepared.json") {
|
||||
continue
|
||||
}
|
||||
var details promptkit.PreparedRun
|
||||
if json.Unmarshal(content, &details) != nil || details.PromptID != "notarius/dnd.scene_descriptions" {
|
||||
continue
|
||||
}
|
||||
for _, message := range details.Messages {
|
||||
if strings.Contains(message.Content, "Roll initiative") {
|
||||
correctedKey = details.MessageContentHash
|
||||
}
|
||||
}
|
||||
}
|
||||
if correctedKey == "" {
|
||||
t.Fatal("no prepared initial combat scene evidence")
|
||||
}
|
||||
corrected := readProductionJSON[dnd.SceneDescriptionList](t, filepath.Join(fixture.cache, correctedKey+".json"))
|
||||
if len(corrected.Scenes) != 1 || corrected.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||
t.Fatalf("cache did not retain the corrected domain candidate: %#v", corrected)
|
||||
}
|
||||
fixture.client.failEvents = false
|
||||
outcome, report, diagnostics := fixture.run(t)
|
||||
if outcome.Status != 0 || report.FallbackCount != 0 || report.ArtifactCount != 12 {
|
||||
t.Fatalf("recovery=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
||||
}
|
||||
after, _ := fixture.client.snapshot()
|
||||
for role, count := range counts {
|
||||
want := count
|
||||
if role == "events" {
|
||||
want++
|
||||
}
|
||||
if after[role] != want {
|
||||
t.Fatalf("recovery reran accepted %s: %v -> %v", role, counts, after)
|
||||
}
|
||||
}
|
||||
artifact := readProductionJSON[dnd.EnemyEventList](t, filepath.Join(report.OutputDirectory, "lanes", "enemy-events.json"))
|
||||
if len(artifact.Events) != 1 || artifact.Events[0].SourceRefs[0].SourceID != "session-ravenfall" {
|
||||
t.Fatalf("recovered domain evidence=%#v", artifact)
|
||||
}
|
||||
stable := readTree(t, fixture.cache)
|
||||
outcome, hitReport, diagnostics := fixture.run(t)
|
||||
if outcome.Status != 0 || hitReport.ArtifactCount != 12 {
|
||||
t.Fatalf("cached run=%#v diagnostics=%q", outcome, diagnostics)
|
||||
}
|
||||
last, _ := fixture.client.snapshot()
|
||||
if !reflect.DeepEqual(last, after) || !sameFiles(stable, readTree(t, fixture.cache)) {
|
||||
t.Fatalf("full hit executed or rewrote acceptance: %v -> %v", after, last)
|
||||
}
|
||||
for name := range readTree(t, hitReport.DebugDirectory) {
|
||||
if strings.Contains(name, "/native/") && strings.HasSuffix(name, "/result.json") {
|
||||
t.Fatalf("full hit fabricated native execution: %s", name)
|
||||
}
|
||||
}
|
||||
hitSummary := readProductionJSON[promptkit.ExecutionGroupSummary](t, filepath.Join(hitReport.DebugDirectory, "executions.json"))
|
||||
if len(hitSummary.Members) != 0 || !hitSummary.TerminalCoverageComplete {
|
||||
t.Fatalf("full hit fabricated historical execution totals: %#v", hitSummary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembledFallbackSkipsDependentReuseAndPreservesFailedRefresh(t *testing.T) {
|
||||
fixture := newRecoveryFixture(t)
|
||||
outcome, _, diagnostics := fixture.run(t)
|
||||
if outcome.Status != 0 {
|
||||
t.Fatalf("initial run=%#v diagnostics=%q", outcome, diagnostics)
|
||||
}
|
||||
before, _ := fixture.client.snapshot()
|
||||
accepted := readTree(t, fixture.cache)
|
||||
fixture.client.failNPC = true
|
||||
description := filepath.Join(filepath.Dir(fixture.config), "pipelines", "dnd-session.describe-session.yml")
|
||||
content, err := os.ReadFile(description)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeConfigKitFile(t, description, replaceRequiredOnce(t, string(content), " extract:\n module: dnd/npc-registry\n", " extract:\n module: dnd/npc-registry\n cache: refresh\n mode: permissive\n"))
|
||||
outcome, report, diagnostics := fixture.run(t)
|
||||
if outcome.Status != 0 || report.FallbackCount != 1 {
|
||||
t.Fatalf("fallback=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
||||
}
|
||||
after, _ := fixture.client.snapshot()
|
||||
for _, role := range []string{"scenes", "kind", "classification", "locations", "items", "location-occurrences", "duplicate_groups"} {
|
||||
if after[role] != before[role] {
|
||||
t.Fatalf("unaffected %s reran: %v -> %v", role, before, after)
|
||||
}
|
||||
}
|
||||
for _, role := range []string{"npcs", "spell_casts", "npc-occurrences", "combat_turns", "events"} {
|
||||
if after[role] <= before[role] {
|
||||
t.Fatalf("affected %s reused earlier acceptance: %v -> %v", role, before, after)
|
||||
}
|
||||
}
|
||||
if !sameFiles(accepted, readTree(t, fixture.cache)) {
|
||||
t.Fatal("failed refresh/fallback replaced accepted cache data")
|
||||
}
|
||||
registry := readProductionJSON[dnd.NPCRegistry](t, filepath.Join(report.OutputDirectory, "lanes", "npc-registry.json"))
|
||||
if registry.NPCs == nil || len(registry.NPCs) != 0 {
|
||||
t.Fatalf("fallback registry=%#v", registry)
|
||||
}
|
||||
fixture.client.failNPC = false
|
||||
writeConfigKitFile(t, description, string(content))
|
||||
outcome, report, diagnostics = fixture.run(t)
|
||||
if outcome.Status != 0 || report.FallbackCount != 0 {
|
||||
t.Fatalf("restored acceptance=%#v report=%#v diagnostics=%q", outcome, report, diagnostics)
|
||||
}
|
||||
last, _ := fixture.client.snapshot()
|
||||
if !reflect.DeepEqual(last, after) {
|
||||
t.Fatalf("failed refresh lost earlier reusable results: %v -> %v", after, last)
|
||||
}
|
||||
}
|
||||
|
||||
var _ promptkit.LLMClient = (*recoveryClient)(nil)
|
||||
Reference in New Issue
Block a user