diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f7d1b064..253c634d 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1666,64 +1666,34 @@ publication then capture failure, useful exit categories, and no reporting sidec prompt filesystem/mount helpers, copied completion/correction/debug contracts, old checkpoint/chunk-plan source packages, and reporting-only module fingerprint methods. No runtime output/cache/debug files were removed. Preserved typed -validation-result checks and intrinsic artifact-schema compatibility; migrated -D&D tests away from retired metadata and intermediate-output surfaces without -relaxing domain codecs. Corrected stale source/JSON fixture expectations and the -configuration NUL/resolved-control assertions. The production executable builds; -the real executable route and text/JSON/YAML/fallback tests now pass with a local -fake HTTP provider, closing their earlier runtime-wrapper setup blocker. All D&D -module and configuration tests, their focused vet checks, contracts/shared/helper -tests, and focused native CLI tests pass. Whole-tree compilation remains pending: -CLI and integration fixtures still reference removed APIs and legacy imports. -Dependency tidy/old-path removal, -notice reconciliation, remaining fixture migration, and full validation are not -complete; this stage is not marked finished. +validation-result checks, intrinsic artifact schemas, domain codecs, reference +compatibility, and source/identity rules. Corrected stale source/JSON fixture +expectations and the configuration NUL/resolved-control assertions. -All framework package tests and vet checks now pass; the complete pipeline -package also passes race checks. Migrated resolution, construction, reference materialization, -generated handoff, codec, source/annotation isolation, diagnostics, and session -coverage to the current contracts. Removed tests tied solely to superseded -outer retries, checkpoint provenance, manifests, and attempt-debug wrappers; -native producer, cache, assessment, and capture tests retain their replacement -behavioral coverage. Whole-tree compile-only validation still fails at the old -ConfigKit YAML import in CLI tests and removed completion/checkpoint APIs in -module integration fixtures; those blockers remain required work. +All framework, module (including the complete integration package), and +configuration tests now compile and pass. Migrated retained resolution, +construction, reference materialization, typed handoff, codec, source/annotation +isolation, diagnostics, and session coverage. Cross-family contracts live in the +black-box module integration package and use real production registration and +native provider fakes; the unchanged import-boundary tests pass. Published domain +and evidence files replace retired manifests and intermediate-output ledgers. +Spell, NPC, location, occurrence, combat, and grounded-pipeline fixtures retain +canonical identity, contextual selection, source chronology, scene eligibility, +complete correction/revalidation, and evidence-union behavior. Invalid combat +enums fail at the native structural boundary rather than bypassing its schema. +Removed tests tied solely to superseded outer retries, checkpoint provenance, +manifests, and attempt-debug wrappers; native producer/cache/assessment/capture +tests own the replacement behavior. Complete pipeline and integration repeated +race checks pass, as do framework/integration vet checks. -Moved the cross-family spell, combat, enemy-event, and Seriatim runner contracts -into the black-box module integration package. They use public module APIs and -share a small native provider/embedded-engine fixture, preserving complete -assessment/history, strict/permissive outcomes, scene gating, intrinsic codec, -and transcript metadata/source-reference coverage. These selected integration -tests and their vet/repeated race checks pass, as do all D&D, generic, and -Seriatim package tests and the unchanged production import-boundary suite. -The remaining legacy integration files still prevent running that whole package; -their completion/checkpoint fixture migration is not finished. - -Migrated the older spell and NPC registry integration fixtures to native -provider requests, real JSON lane publication, and version 6 test configurations. -Kept source ordering/identity, capability failures, reference inputs, malformed -candidate rejection, and consolidation coverage. NPC reconciliation now tests -explicit strict failure, permissive final safe groups with one qualified fallback, -and correction to a complete proposal. Shared production registries remain real; -test providers classify the selected native response contract rather than -recreating completion requests or manually decoding model output into modules. -These selected integration tests, vet/race checks, all framework/configuration -tests, and import-boundary tests pass. Remaining combat, location, occurrence, -and grounded-pipeline fixtures still block full integration compilation. - -NPC-occurrence and location handoff tests now run through native providers and -published lane/evidence files. Retained contextual same-name location selection, -names-only NPC projection, deterministic durable IDs, source chronology, earlier- -producer requirements, and no consumer execution after strict producer rejection. -The multi-chunk semantic NPC fixture now derives evidence from the actual rendered -chunk rather than completion order, and verifies both downstream chunks with the -canonical registry identity under ordinary parallel workers. Removed only the old -generated-reference checkpoint assertion; existing native cache/eligibility tests -own its replacement behavior. Updated occurrence test configurations to version 6 -and made temporary config cleanup test-owned. These selected integration tests, -vet/repeated race checks, framework/configuration tests, and import-boundary tests -pass. Only combat and grounded-pipeline legacy fixtures remain as integration -compile blockers; CLI/dependency cleanup and complete validation are still pending. +The production executable builds. Real executable route and text/JSON/YAML/ +fallback checks passed with a local fake HTTP provider after removal of the old +runtime wrapper. Focused native CLI tests pass, but the full CLI suite and +whole-tree compilation remain blocked by legacy CLI fixtures/imports; the current +setup failure is old-path ConfigKit importing the removed YAML `plugin/limit`. +Remaining CLI test migration, dependency tidy/old-path removal, notice +reconciliation, and whole-tree validation are required before completion. +This stage is not marked finished. **Goal:** delete the old architecture rather than leave dormant alternatives. diff --git a/internal/modules/integration/dnd_combat_runner_test.go b/internal/modules/integration/dnd_combat_runner_test.go index fc14fe9e..deec7b9a 100644 --- a/internal/modules/integration/dnd_combat_runner_test.go +++ b/internal/modules/integration/dnd_combat_runner_test.go @@ -2,7 +2,7 @@ package integration_test import ( "context" - "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -23,11 +23,11 @@ import ( combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity" npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry" - combatshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape" "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" + "go.maximumdirect.net/promptkit" ) -func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing.T) { +func TestProductionCombatPipelineCorrectsMergesNormalizesAndWritesJSON(t *testing.T) { registries := productionNPCRegistries(t) catalog := moduleCatalog(registries) npcPayload := combatTestNPCPayload(t) @@ -49,23 +49,15 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing } client := &fakeCombatLLMClient{responses: []string{ - combatTestInvalidEnumResponse("unsupported"), + combatTestTurnResponse("", "turn", 1), combatTestTurnResponse("mira thorn", "turn", 1), combatTestTurnResponse("Mira Thorn", "reaction", 3), combatTestTurnResponse("Hooded Guard", "turn", 5), }} - prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatalf("Prepare() error = %v, want nil", err) } - fingerprints := prepared.CheckpointFingerprints() - assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":npc_registry") - assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":scene_eligibility") - assertCombatFingerprint(t, fingerprints, "normalize:combat:"+combatnormalize.Key+":npc_registry") - if len(fingerprints) == 0 { - t.Fatal("checkpoint fingerprints = empty, want production combat identities") - } - output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ Prepared: prepared, RawInput: readNPCFixture(t), @@ -74,20 +66,14 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing if err != nil { t.Fatalf("Run() error = %v, want nil", err) } - if len(client.requests) != 4 || len(output.Rejected) != 0 { - t.Fatalf("LLM requests = %d rejected = %#v, want one retry and no durable rejection", len(client.requests), output.Rejected) + if len(client.requests) != 4 || len(output.OutputFiles) != 1 || output.FallbackCount != 0 { + t.Fatalf("native calls=%d output=%#v, want corrected accepted combat", len(client.requests), output) } - if output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 || len(output.OutputFiles) == 0 { - t.Fatalf("manifest/output = %#v / %#v, want approved combat JSON output", output.Manifest, output.OutputFiles) - } - serialized := output.NormalizeOutputs[0] - if serialized.LaneID != "combat" || serialized.NormalizerKey != combatnormalize.Key || serialized.Artifact.Schema.ID != combatcodec.SchemaID { - t.Fatalf("serialized combat output = %#v, want production combat lane schema", serialized) - } - value, err := combatcodec.New().Decode(serialized.Artifact.Content) + value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes) if err != nil { - t.Fatalf("Decode(combat output) error = %v", err) + t.Fatal(err) } + if len(value.CombatTurns) != 3 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[1].Actor != "Mira Thorn" || value.CombatTurns[2].Actor != "Hooded Guard" { t.Fatalf("normalized combat output = %#v, want ordered canonical actors", value) } @@ -101,21 +87,6 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing if !hasCombatWarning(output.Diagnostics, combatnormalize.ReasonCodeActorCanonicalized) { t.Fatalf("diagnostics = %#v, want registry normalization advisory", output.Diagnostics) } - if len(output.Manifest.References) != 3 { - t.Fatalf("manifest references = %#v, want scene eligibility plus extract and normalize NPC provenance", output.Manifest.References) - } - lane := output.Manifest.ArtifactLanes[0] - if lane.ID != "combat" || lane.Extractor != combatextract.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != combatnormalize.Key { - t.Fatalf("manifest lane = %#v, want complete combat composition", lane) - } - extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) - if !ok || extractorMetadata["npc_count"] != 2 || extractorMetadata["npc_registry_digest"] == "" || extractorMetadata["scene_description_count"] != 3 || extractorMetadata["scene_eligibility_digest"] == "" || extractorMetadata["scene_gate_policy"] != "dnd.combat_turns.scene_gate.v1" { - t.Fatalf("extractor metadata = %#v, want registry and scene eligibility metadata", lane.Metadata) - } - normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any) - if !ok || normalizerMetadata["npc_count"] != 2 || normalizerMetadata["normalization_policy"] != combatnormalize.NormalizationPolicy { - t.Fatalf("normalizer metadata = %#v, want registry and normalization policy", lane.Metadata) - } var combatFile *contracts.OutputFile for index := range output.OutputFiles { if output.OutputFiles[index].Name == "lanes/combat.json" { @@ -128,7 +99,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing } } -func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidation(t *testing.T) { +func TestProductionCombatPipelineStopsAfterNativeStructuralFailure(t *testing.T) { registries := productionNPCRegistries(t) configValue := combatOnlyConfig() profile := configValue.Pipelines["dnd-combat-fixture"] @@ -148,17 +119,12 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio if err != nil { t.Fatalf("Resolve() error = %v, want nil", err) } - client := &fakeCombatLLMClient{responses: []string{ - combatTestInvalidEnumResponse("unsupported"), - combatTestInvalidEnumResponse("invalid"), - combatTestInvalidEnumResponse("unknown"), - }} + client := &fakeCombatLLMClient{response: combatTestInvalidEnumResponse("unsupported")} materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, moduleCatalog(registries), pipeline.ReferenceMaterializationOptions{}) if err != nil { t.Fatalf("MaterializeReferences() error = %v", err) } - materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput - prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatalf("Prepare() error = %v, want nil", err) } @@ -167,18 +133,12 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio RawInput: readNPCFixture(t), Workers: 1, }) - if err != nil { - t.Fatalf("Run() error = %v, want non-fatal rejected output", err) + + if !errors.Is(err, promptkit.ErrValidation) { + t.Fatalf("Run() error=%v, want native structural failure", err) } - if len(client.requests) != 3 || len(output.Rejected) != 1 { - t.Fatalf("LLM requests = %d rejected = %#v, want exhausted retry and one rejection", len(client.requests), output.Rejected) - } - rejected := output.Rejected[0] - if rejected.ReasonCode != combatshape.ReasonCode || rejected.ValidatorName != combatshape.Key || rejected.AttemptCount != 3 { - t.Fatalf("rejected output = %#v, want exhausted combat shape rejection", rejected) - } - if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 { - t.Fatalf("output = %#v, want non-fatal rejected combat result without normalized artifacts", output) + if len(client.requests) == 0 || len(output.OutputFiles) != 0 { + t.Fatalf("native calls=%d output=%#v, want exhausted rejection", len(client.requests), output) } } @@ -205,7 +165,7 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) } client := &fakeCombatLLMClient{} - prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatalf("Prepare() error = %v", err) } @@ -217,13 +177,10 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t if err != nil { t.Fatalf("Run() error = %v", err) } - if len(client.requests) != 0 || len(output.Rejected) != 0 || !hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") { - t.Fatalf("requests/rejected/diagnostics = %#v / %#v / %#v, want accepted unavailable-scene skip without a combat request", client.requests, output.Rejected, output.Diagnostics) + if len(client.requests) != 0 || !hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") { + t.Fatalf("native calls=%d diagnostics=%#v, want unavailable-scene skip", len(client.requests), output.Diagnostics) } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("normalized outputs = %#v, want accepted empty combat lane", output.NormalizeOutputs) - } - value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content) + value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes) if err != nil { t.Fatalf("Decode(combat output) error = %v", err) } @@ -297,7 +254,7 @@ func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecutio t.Fatalf("MaterializeReferences() error = %v, want materialization before typed preparation", err) } client := &fakeCombatLLMClient{} - if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}); err == nil { + if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}); err == nil { t.Fatal("Prepare() error = nil, want NPC registry preparation failure") } if len(client.requests) != 0 { @@ -352,7 +309,7 @@ func combatOnlyConfig() config.Config { References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference(repositoryPathForIntegration("internal", "modules", "integration", "testdata", "combat_scene_descriptions.json"))}, Artifacts: map[string]pipeline.ArtifactLaneProfile{ "combat": { - Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2}, + Extract: pipeline.ModuleBinding{Module: combatextract.Key}, Normalize: pipeline.Binding(combatnormalize.Key), }, }, @@ -368,30 +325,26 @@ func repositoryPathForIntegration(parts ...string) string { type fakeCombatLLMClient struct { mu sync.Mutex + response string responses []string - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest } -func (client *fakeCombatLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (client *fakeCombatLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - snapshot, err := contracts.CloneStructuredCompletionRequest(req) - if err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err) + return nil, err } client.mu.Lock() - client.requests = append(client.requests, snapshot) + client.requests = append(client.requests, req) index := len(client.requests) - 1 client.mu.Unlock() + if client.response != "" { + return &promptkit.GenerateResponse{Content: client.response}, nil + } if index >= len(client.responses) { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("no combat fake response for call %d", index) + return nil, fmt.Errorf("no combat fake response for call %d", index) } - content := []byte(client.responses[index]) - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate combat fake target: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "combat-fake"}, nil + return &promptkit.GenerateResponse{Content: client.responses[index]}, nil } func combatTestInvalidEnumResponse(turnKind string) string { @@ -411,14 +364,4 @@ func hasCombatWarning(diagnostics contracts.DiagnosticCollection, reason string) return false } -func assertCombatFingerprint(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) { - t.Helper() - for _, fingerprint := range fingerprints { - if fingerprint.Name == name && strings.HasPrefix(fingerprint.Value, "sha256:") { - return - } - } - t.Fatalf("fingerprints = %#v, want %q with semantic digest", fingerprints, name) -} - -var _ contracts.StructuredLLMClient = (*fakeCombatLLMClient)(nil) +var _ promptkit.LLMClient = (*fakeCombatLLMClient)(nil) diff --git a/internal/modules/integration/dnd_npc_grounded_test.go b/internal/modules/integration/dnd_npc_grounded_test.go index e4c4ed47..3a27a420 100644 --- a/internal/modules/integration/dnd_npc_grounded_test.go +++ b/internal/modules/integration/dnd_npc_grounded_test.go @@ -2,7 +2,6 @@ package integration_test import ( "context" - "crypto/sha256" "encoding/json" "fmt" "os" @@ -14,21 +13,14 @@ import ( "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns" npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" - spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells" - combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry" - sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" - npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry" combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics" + "go.maximumdirect.net/promptkit" ) func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) { @@ -38,233 +30,93 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T campaignInputs := configureGroundedCampaignReferences(t, &configValue) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) if err != nil { - t.Fatalf("Resolve() error = %v", err) + t.Fatal(err) } materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{}) if err != nil || len(warnings) != 0 { - t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) + t.Fatalf("materialize references: %v warnings=%#v", err, warnings) } - - client := &groundedDNDLLMClient{} - prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) - if err != nil { - t.Fatalf("Prepare() error = %v", err) - } - for name, value := range map[string]string{ - "extract:npc_registry:dnd/npc-registry:mapping_policy": "dnd.npc_registry.extract_mapping.v3", - "normalize:npc_registry:dnd/npc-registry:identity_policy": "dnd.npc_registry.identity.v1", - "normalize:npc_registry:dnd/npc-registry:normalization_policy": npcnormalize.NormalizationPolicy, - "normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_policy": semanticreconcile.Policy, - "extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v3", - "extract:combat:dnd/combat-turns:scene_gate_policy": "dnd.combat_turns.scene_gate.v1", - } { - assertFingerprintValue(t, prepared.CheckpointFingerprints(), name, value) - } - for _, name := range []string{ - "extract:npc_registry:dnd/npc-registry:prompt", - "extract:npc_registry:dnd/npc-registry:response_schema", - "normalize:npc_registry:dnd/npc-registry:prompt", - "normalize:npc_registry:dnd/npc-registry:response_schema", - "normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_limits", - "extract:spells:dnd/spells:prompt", - "extract:spells:dnd/spells:response_schema", - "extract:spells:dnd/spells:npc_registry", - "extract:combat:dnd/combat-turns:prompt", - "extract:combat:dnd/combat-turns:response_schema", - "extract:combat:dnd/combat-turns:scene_eligibility", - "extract:combat:dnd/combat-turns:npc_registry", - "normalize:combat:dnd/combat-turns:npc_registry", - } { - assertCombatFingerprint(t, prepared.CheckpointFingerprints(), name) - } - checkpoint := newGeneratedReferenceCheckpointLoader() - output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: readNPCFixture(t), - Workers: 1, - Checkpoint: checkpoint, - }) - if err != nil { - t.Fatalf("Run() error = %v", err) - } - if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 4 { - t.Fatalf("run outputs = %#v rejected = %#v, want NPC, scene, spell, and combat outputs", output.NormalizeOutputs, output.Rejected) - } - wantSchemas := map[string]string{"npc_registry": npccodec.SchemaID, "scene-descriptions": scenecodec.SchemaID, "spells": spellcodec.SchemaID, "combat": combatcodec.SchemaID} - for _, serialized := range output.NormalizeOutputs { - if serialized.Artifact.Schema.ID != wantSchemas[serialized.LaneID] || serialized.Artifact.Schema.Version != "v1" { - t.Fatalf("%s artifact schema = %#v, want minimal v1 identity", serialized.LaneID, serialized.Artifact.Schema) - } - } - wantExtractorIdentity := map[string]struct{ promptID, schemaID string }{ - "npc_registry": {npcregistry.PromptID, npcregistry.ResponseSchemaID}, - "scene-descriptions": {sceneextract.PromptID, sceneextract.ResponseSchemaID}, - "spells": {spells.PromptID, spells.ResponseSchemaID}, - "combat": {combatextract.PromptID, combatextract.ResponseSchemaID}, - } - for _, lane := range output.Manifest.ArtifactLanes { - want, ok := wantExtractorIdentity[lane.ID] - metadata, metadataOK := lane.Metadata["extractor"].(map[string]any) - if !ok || !metadataOK || metadata["prompt_id"] != want.promptID || metadata["prompt_version"] != "v1" || metadata["response_schema_id"] != want.schemaID || metadata["response_schema_version"] != "v1" { - t.Fatalf("%s extractor metadata = %#v, want v1 prompt/schema identity", lane.ID, metadata) - } - } - - var npcPayload, scenePayload []byte - seenSteps := map[string]string{} - for _, serialized := range output.NormalizeOutputs { - seenSteps[serialized.LaneID] = serialized.StepID - switch serialized.LaneID { - case "npc_registry": - npcPayload = append([]byte(nil), serialized.Artifact.Content...) - case "scene-descriptions": - scenePayload = append([]byte(nil), serialized.Artifact.Content...) - } - } - if seenSteps["npc_registry"] != "identify-npcs" || seenSteps["scene-descriptions"] != "identify-npcs" || seenSteps["spells"] != "grounded-events" || seenSteps["combat"] != "grounded-events" { - t.Fatalf("normalized output steps = %#v, want ordered producer and consumer steps", seenSteps) - } - if len(npcPayload) == 0 { - t.Fatal("NPC producer did not publish a canonical payload") - } - if len(scenePayload) == 0 { - t.Fatal("scene producer did not publish a canonical payload") - } - npcValue, err := npccodec.New().Decode(npcPayload) - if err != nil { - t.Fatalf("decode NPC producer payload: %v", err) - } - npcPayload, err = npccodec.New().Encode(npcValue) - if err != nil { - t.Fatalf("encode canonical NPC producer payload: %v", err) - } - canonicalDigest := digestBytes(npcPayload) - sceneValue, err := scenecodec.New().Decode(scenePayload) - if err != nil { - t.Fatalf("decode scene producer payload: %v", err) - } - scenePayload, err = scenecodec.New().Encode(sceneValue) - if err != nil { - t.Fatalf("encode canonical scene producer payload: %v", err) - } - sceneDigest := digestBytes(scenePayload) - projection := []byte(`{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`) - projectionDigest := digestBytes(projection) - seenConsumers := map[string]bool{} - for _, request := range client.requestsSnapshot() { - if request.PromptID != spells.PromptID && request.PromptID != combatextract.PromptID { - continue - } - input := request.Inputs["npc_registry"] - if input.MediaType != npccodec.MediaType || input.Digest != projectionDigest || string(input.Content) != string(projection) || input.OriginURI != "" { - t.Fatalf("%s NPC prompt input = %#v, want names-only generated registry", request.PromptID, input) - } - if request.PromptID == combatextract.PromptID { - transcript, ok := request.Inputs["transcript"] - if !ok || !strings.Contains(string(transcript.Content), "Mira Thorn asks the party") { - t.Fatalf("combat transcript input = %#v, want configured transcript content", transcript) - } - for name, want := range campaignInputs { - input, ok := request.Inputs[name] - if !ok || string(input.Content) != want { - t.Fatalf("combat %s input = %#v, want configured campaign reference", name, input) - } - } - if _, ok := request.Inputs["scene_descriptions"]; ok { - t.Fatalf("combat prompt inputs = %#v, must not contain scene descriptions", request.Inputs) - } - } - seenConsumers[request.PromptID] = true - } - if !seenConsumers[spells.PromptID] || !seenConsumers[combatextract.PromptID] { - t.Fatalf("consumer prompt IDs = %#v, want spell and combat requests", seenConsumers) - } - if combatCompletionCount(client.requestsSnapshot()) != 1 { - t.Fatalf("combat completion requests = %#v, want exactly one for the exact combat scene", client.requestsSnapshot()) - } - - provenanceCount := 0 - for _, reference := range output.Manifest.References { - if reference.SlotName != "npc_registry" { - continue - } - provenanceCount++ - if reference.Digest != canonicalDigest || reference.OriginType != "generated" { - t.Fatalf("NPC generated provenance = %#v, want canonical digest and generated origin", reference) - } - } - if provenanceCount != 3 { - t.Fatalf("NPC generated provenance count = %d, want spell extract plus combat extract/normalize", provenanceCount) - } - sceneProvenanceCount := 0 - for _, reference := range output.Manifest.References { - if reference.SlotName == "scene_descriptions" { - sceneProvenanceCount++ - if reference.Stage != string(pipeline.StageExtract) || reference.StepID != "grounded-events" || reference.LaneID != "combat" || reference.OriginType != "generated" || reference.Digest != sceneDigest || reference.ArtifactKind != string(dnd.SceneDescriptionListKind) || reference.SchemaID != scenecodec.SchemaID || reference.SchemaName != scenecodec.SchemaName || reference.SchemaVersion != scenecodec.SchemaVersion || reference.MediaType != scenecodec.MediaType || reference.ProducerPipeline != "dnd-npc-grounded" || reference.ProducerStep != "identify-npcs" || reference.ProducerLane != "scene-descriptions" || reference.ProducerModule != "dnd/scene-descriptions" { - t.Fatalf("scene generated provenance = %#v, want canonical combat handoff identity", reference) - } - } - } - if sceneProvenanceCount != 1 { - t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount) - } - assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat")) - manifestContent, err := json.Marshal(output.Manifest) + client := &groundedDNDLLMClient{sceneTitle: "eligibility-only-scene-sentinel"} + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatal(err) } - for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons", "A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} { - if strings.Contains(string(manifestContent), forbidden) { - t.Fatalf("manifest exposes prompt or reference payload content %q", forbidden) - } + output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readNPCFixture(t), Workers: 1}) + if err != nil { + t.Fatal(err) } - for _, lane := range output.Manifest.ArtifactLanes { - componentContent, err := json.Marshal(lane.Metadata) - if err != nil { - t.Fatalf("marshal %s component metadata: %v", lane.ID, err) - } - for _, forbidden := range []string{"A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} { - if strings.Contains(string(componentContent), forbidden) { - t.Fatalf("%s component metadata exposes scene or transcript content %q", lane.ID, forbidden) - } - } - for _, component := range []string{"extractor", "normalizer"} { - metadata, ok := lane.Metadata[component].(map[string]any) - if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) { - t.Fatalf("%s %s metadata = %#v, want generated identity only in framework provenance", lane.ID, component, metadata) - } - } + if len(output.OutputFiles) != 4 || output.FallbackCount != 0 { + t.Fatalf("output=%#v, want four accepted domain artifacts", output) } - var combatValue dnd.CombatTurnList - for _, serialized := range output.NormalizeOutputs { - switch serialized.LaneID { - case "spells": - spellValue := decodeRunnerSpellResponse(t, serialized.Artifact.Content) - if len(spellValue.SpellCasts) != 1 || spellValue.SpellCasts[0].Caster != "Mira Thorn" { - t.Fatalf("spell output = %#v, want one registry-grounded-context spell", spellValue) - } - assertCurrentEvidence(t, spellValue.SpellCasts[0].SourceRefs) - case "combat": - decoded, decodeErr := combatcodec.New().Decode(serialized.Artifact.Content) - if decodeErr != nil { - t.Fatalf("decode combat output: %v", decodeErr) - } - combatValue = decoded - } + npcs, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes) + if err != nil || len(npcs.NPCs) != 2 { + t.Fatalf("NPC registry=%#v error=%v", npcs, err) } + scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes) + if err != nil || len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat { + t.Fatalf("scene descriptions=%#v error=%v", scenes, err) + } + + projection := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}` + seenConsumers := map[string]bool{} + for _, request := range client.requestsSnapshot() { + kind := "spell_casts" + if nativeSchemaHasProperty(request, "combat_turns") { + kind = "combat_turns" + } else if !nativeSchemaHasProperty(request, "spell_casts") { + continue + } + input, err := nativeJSONInput(request, "npcs") + if err != nil || string(input) != projection { + t.Fatalf("%s NPC projection=%s error=%v", kind, input, err) + } + text := nativePromptText(request) + for _, npc := range npcs.NPCs { + if strings.Contains(text, npc.ID) { + t.Fatalf("%s prompt contains opaque registry identity", kind) + } + } + if kind == "combat_turns" { + transcript, err := nativeJSONInput(request, "units") + if err != nil || !strings.Contains(string(transcript), "Mira Thorn asks the party") { + t.Fatalf("combat transcript=%s error=%v", transcript, err) + } + for name, want := range campaignInputs { + if !strings.Contains(text, want) { + t.Fatalf("combat prompt omitted %s reference", name) + } + } + for _, excluded := range []string{client.sceneTitle, "The party faces an active encounter."} { + if strings.Contains(text, excluded) { + t.Fatalf("combat prompt included eligibility-only scene prose %q", excluded) + } + } + } + seenConsumers[kind] = true + } + if !seenConsumers["spell_casts"] || !seenConsumers["combat_turns"] || combatCompletionCount(client.requestsSnapshot()) != 1 { + t.Fatalf("consumer native requests=%#v", seenConsumers) + } + spellsValue := decodeRunnerSpellResponse(t, publishedLane(t, output, "spells").Bytes) + if len(spellsValue.SpellCasts) != 1 || spellsValue.SpellCasts[0].Caster != "Mira Thorn" { + t.Fatalf("spells=%#v", spellsValue) + } + assertCurrentEvidence(t, spellsValue.SpellCasts[0].SourceRefs) + combatValue := publishedCombatOutput(t, output) if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" { - t.Fatalf("combat output = %#v, want registry-normalized actor", combatValue) + t.Fatalf("combat=%#v", combatValue) } assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs) } -func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) { +func TestSceneCombatSemanticsCorrectionRevalidatesRevisedScene(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) profile := configValue.Pipelines["dnd-npc-grounded"] sceneLane := profile.Steps[0].Artifacts["scene-descriptions"] - sceneLane.Extract.Retries = 1 + turns := 1 + sceneLane.Extract.MaxAdditionalTurns = &turns sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{ pipeline.Binding("generic/valid_json"), pipeline.Binding("extract/dnd/scene-descriptions/shape"), @@ -277,48 +129,48 @@ func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) { configValue.Pipelines["dnd-npc-grounded"] = profile client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}} - output := runGroundedPipeline(t, configValue, registries, client, nil) - if len(output.Rejected) != 0 { - t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected) + output := runGroundedPipeline(t, configValue, registries, client) + if output.FallbackCount != 0 { + t.Fatalf("unexpected fallback: %#v", output) } for _, group := range output.Diagnostics.Groups { if group.Origin.ValidatorKey == combatsemantics.Key { t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group) } } - var scenes dnd.SceneDescriptionList - for _, serialized := range output.NormalizeOutputs { - if serialized.LaneID == "scene-descriptions" { - var err error - scenes, err = scenecodec.New().Decode(serialized.Artifact.Content) - if err != nil { - t.Fatal(err) - } - } + scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes) + if err != nil { + t.Fatal(err) } + if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat { t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes) } requests := client.requestsSnapshot() - var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest + var sceneRequests, validatorRequests []promptkit.GenerateRequest for _, request := range requests { - switch request.PromptID { - case sceneextract.PromptID: + switch { + case nativeSchemaHasProperty(request, "kind"): sceneRequests = append(sceneRequests, request) - case combatsemantics.PromptID: + case nativeSchemaHasProperty(request, "classification"): validatorRequests = append(validatorRequests, request) } } - if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil { - t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests)) + if len(sceneRequests) != 2 || len(validatorRequests) != 2 { + t.Fatalf("scene/validator calls=%d/%d, want correction and revalidation", len(sceneRequests), len(validatorRequests)) } - correction := sceneRequests[1].Correction - if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") { - t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction) + original := sceneRequests[0].Prompt.Messages + revised := sceneRequests[1].Prompt.Messages + if len(revised) != len(original)+2 || !reflect.DeepEqual(revised[:len(original)], original) { + t.Fatal("scene correction lost native conversation history") + } + assistant, guidance := revised[len(original)], revised[len(original)+1] + if assistant.Role != promptkit.RoleAssistant || !strings.Contains(assistant.Content, `"kind":"narrative"`) || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "Return kind: combat") || !strings.Contains(guidance.Content, "active encounter") { + t.Fatalf("native correction messages=%#v", revised[len(original):]) } for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} { - if strings.Contains(correction.UserGuidance, forbidden) { - t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance) + if strings.Contains(guidance.Content, forbidden) { + t.Fatalf("correction guidance contains diagnostic identifier %q", forbidden) } } } @@ -338,7 +190,7 @@ func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) { `"id": 4`, `"id": 50`, `"id": 5`, `"id": 40`, ).Replace(string(readNPCFixture(t))) - output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, nil, []byte(raw)) + output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, []byte(raw)) value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json")) if err != nil { @@ -357,7 +209,7 @@ func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testi "enabled": true, "lanes": []any{"scene-descriptions"}, }} configValue.Pipelines["dnd-npc-grounded"] = profile - output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}, nil) + output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}) value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json")) if err != nil { @@ -379,45 +231,20 @@ func evidenceUnitIDs(value evidencecontext.Document) []int { func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) { registries := productionNPCRegistries(t) configValue := loadGroundedPipelineConfig(t) - profile := configValue.Pipelines["dnd-npc-grounded"] - combat := profile.Steps[1].Artifacts["combat"] - combat.Extract.Retries = 2 - profile.Steps[1].Artifacts["combat"] = combat - configValue.Pipelines["dnd-npc-grounded"] = profile - client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative} - output := runGroundedPipeline(t, configValue, registries, client, nil) + output := runGroundedPipeline(t, configValue, registries, client) if combatCompletionCount(client.requestsSnapshot()) != 0 { t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot()) } - if len(output.Rejected) != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") { - t.Fatalf("narrative output rejected/diagnostics = %#v / %#v, want accepted skip without unavailable advisory", output.Rejected, output.Diagnostics) + if output.FallbackCount != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") { + t.Fatalf("narrative output=%#v, want accepted skip", output) } - combatValue := normalizedCombatOutput(t, output) + combatValue := publishedCombatOutput(t, output) if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 { t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue) } } -func TestGeneratedSceneReferenceChangesCombatCheckpointDependency(t *testing.T) { - registries := productionNPCRegistries(t) - first := generatedSceneDependency(t, registries, "A combat encounter") - second := generatedSceneDependency(t, registries, "A revised combat encounter") - if first == second { - t.Fatalf("generated combat dependencies = %q and %q, want canonical scene artifact change to invalidate reuse", first, second) - } -} - -func assertFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, want string) { - t.Helper() - for _, fingerprint := range fingerprints { - if fingerprint.Name == name && fingerprint.Value == want { - return - } - } - t.Fatalf("fingerprints = %#v, want %q = %q", fingerprints, name, want) -} - func assertCurrentEvidence(t *testing.T, references []source.SourceRef) { t.Helper() for _, reference := range references { @@ -451,11 +278,11 @@ func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[s return values } -func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput { - return runGroundedPipelineWithRaw(t, configValue, registries, client, checkpoint, readNPCFixture(t)) +func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient) pipeline.RunOutput { + return runGroundedPipelineWithRaw(t, configValue, registries, client, readNPCFixture(t)) } -func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader, raw []byte) pipeline.RunOutput { +func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, raw []byte) pipeline.RunOutput { t.Helper() catalog := moduleCatalog(registries) effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog}) @@ -466,15 +293,14 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr if err != nil || len(warnings) != 0 { t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings) } - prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}) + prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}) if err != nil { t.Fatalf("Prepare() error = %v", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ - Prepared: prepared, - RawInput: append([]byte(nil), raw...), - Workers: 1, - Checkpoint: checkpoint, + Prepared: prepared, + RawInput: append([]byte(nil), raw...), + Workers: 1, }) if err != nil { t.Fatalf("Run() error = %v", err) @@ -482,80 +308,28 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr return output } -func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { +func publishedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList { t.Helper() - for _, serialized := range output.NormalizeOutputs { - if serialized.LaneID != "combat" { - continue - } - value, err := combatcodec.New().Decode(serialized.Artifact.Content) - if err != nil { - t.Fatalf("decode combat output: %v", err) - } - return value + value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes) + if err != nil { + t.Fatal(err) } - t.Fatalf("normalized outputs = %#v, want combat lane", output.NormalizeOutputs) - return dnd.CombatTurnList{} + return value } -func combatCompletionCount(requests []contracts.StructuredCompletionRequest) int { +func combatCompletionCount(requests []promptkit.GenerateRequest) int { count := 0 for _, request := range requests { - if request.PromptID == combatextract.PromptID { + if nativeSchemaHasProperty(request, "combat_turns") { count++ } } return count } -func generatedSceneDependency(t *testing.T, registries pipeline.Registries, sceneTitle string) string { - t.Helper() - client := &groundedDNDLLMClient{sceneTitle: sceneTitle} - checkpoint := newGeneratedReferenceCheckpointLoader() - runGroundedPipeline(t, loadGroundedPipelineConfig(t), registries, client, checkpoint) - return assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat")) -} - -func assertGeneratedSceneDependency(t *testing.T, dependencies []pipeline.CheckpointFingerprint) string { - t.Helper() - for _, dependency := range dependencies { - if dependency.Name == "generated-reference:scene_descriptions:0" && strings.HasPrefix(dependency.Value, "sha256:") { - return dependency.Value - } - } - t.Fatalf("combat extract checkpoint dependencies = %#v, want generated scene reference fingerprint", dependencies) - return "" -} - -type generatedReferenceCheckpointLoader struct { - pipeline.CheckpointLoader - extract map[string][][]pipeline.CheckpointFingerprint -} - -func newGeneratedReferenceCheckpointLoader() *generatedReferenceCheckpointLoader { - return &generatedReferenceCheckpointLoader{ - CheckpointLoader: pipeline.NoopCheckpointLoader(), - extract: make(map[string][][]pipeline.CheckpointFingerprint), - } -} - -func (loader *generatedReferenceCheckpointLoader) Enabled() bool { return true } - -func (loader *generatedReferenceCheckpointLoader) Extract(laneID, _ string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) { - loader.extract[laneID] = append(loader.extract[laneID], append([]pipeline.CheckpointFingerprint(nil), dependencies...)) - return pipeline.ExtractCheckpoint{}, pipeline.NewCheckpointDecision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing) -} - -func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID string) []pipeline.CheckpointFingerprint { - if len(loader.extract[laneID]) != 1 { - return nil - } - return loader.extract[laneID][0] -} - type groundedDNDLLMClient struct { mu sync.Mutex - requests []contracts.StructuredCompletionRequest + requests []promptkit.GenerateRequest sceneKind dnd.SceneKind sceneTitle string firstUnitID int @@ -566,13 +340,13 @@ type groundedDNDLLMClient struct { combatSemanticsCalls int } -func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { +func (client *groundedDNDLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { if err := ctx.Err(); err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } client.mu.Lock() - client.requests = append(client.requests, cloneStructuredCompletionRequest(request)) - client.mu.Unlock() + defer client.mu.Unlock() + client.requests = append(client.requests, request) var payload any firstUnitID := client.firstUnitID @@ -583,8 +357,8 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ if thirdUnitID == 0 { thirdUnitID = 3 } - switch request.PromptID { - case npcregistry.PromptID: + switch { + case nativeSchemaHasProperty(request, "npcs"): payload = map[string]any{"npcs": []any{ map[string]any{ "name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, @@ -593,9 +367,9 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ "name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}}, }, }} - case npcnormalize.PromptID: + case nativeSchemaHasProperty(request, "duplicate_groups"): payload = map[string]any{"duplicate_groups": []any{}} - case sceneextract.PromptID: + case nativeSchemaHasProperty(request, "kind"): kind := client.sceneKind if client.sceneCalls < len(client.sceneKinds) { kind = client.sceneKinds[client.sceneCalls] @@ -609,46 +383,38 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ title = "A combat encounter" } payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."} - case combatsemantics.PromptID: + case nativeSchemaHasProperty(request, "classification"): classification := "combat" if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) { classification = client.combatSemanticsClassifications[client.combatSemanticsCalls] } client.combatSemanticsCalls++ payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."} - case spells.PromptID: + case nativeSchemaHasProperty(request, "spell_casts"): payload = map[string]any{"spell_casts": []any{map[string]any{ "caster": "Mira Thorn", "spell": "Cure Wounds", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, }}} - case combatextract.PromptID: + case nativeSchemaHasProperty(request, "combat_turns"): payload = map[string]any{"combat_turns": []any{map[string]any{ "actor": "Mira Thorn", "turn_kind": "turn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}}, }}} default: - return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected grounded prompt %q", request.PromptID) + return nil, fmt.Errorf("unexpected native grounded response contract") } content, err := json.Marshal(payload) if err != nil { - return contracts.StructuredCompletionResponse{}, err + return nil, err } - if err := json.Unmarshal(content, out); err != nil { - return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate grounded response: %w", err) - } - return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "grounded-fake"}, nil + return &promptkit.GenerateResponse{Content: string(content)}, nil } -func (client *groundedDNDLLMClient) requestsSnapshot() []contracts.StructuredCompletionRequest { +func (client *groundedDNDLLMClient) requestsSnapshot() []promptkit.GenerateRequest { client.mu.Lock() defer client.mu.Unlock() - return append([]contracts.StructuredCompletionRequest(nil), client.requests...) + return append([]promptkit.GenerateRequest(nil), client.requests...) } -func digestBytes(content []byte) string { - sum := sha256.Sum256(content) - return fmt.Sprintf("sha256:%x", sum[:]) -} - -var _ contracts.StructuredLLMClient = (*groundedDNDLLMClient)(nil) +var _ promptkit.LLMClient = (*groundedDNDLLMClient)(nil) diff --git a/internal/modules/integration/testdata/dnd_npc_grounded_pipeline.yml b/internal/modules/integration/testdata/dnd_npc_grounded_pipeline.yml index 035dd4de..7a438fe2 100644 --- a/internal/modules/integration/testdata/dnd_npc_grounded_pipeline.yml +++ b/internal/modules/integration/testdata/dnd_npc_grounded_pipeline.yml @@ -1,10 +1,6 @@ -version: 5 +version: 6 output: directory: ./notarius-output -cache: - chunk_plans: - mode: bypass - checkpoints: {} debug: directory: ./notarius-debug pipelines: