Migrate combat and grounded pipeline integration tests to native execution
This commit is contained in:
@@ -2,7 +2,7 @@ package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -23,11 +23,11 @@ import (
|
||||
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
||||
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
|
||||
combatshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing.T) {
|
||||
func TestProductionCombatPipelineCorrectsMergesNormalizesAndWritesJSON(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
catalog := moduleCatalog(registries)
|
||||
npcPayload := combatTestNPCPayload(t)
|
||||
@@ -49,23 +49,15 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
}
|
||||
|
||||
client := &fakeCombatLLMClient{responses: []string{
|
||||
combatTestInvalidEnumResponse("unsupported"),
|
||||
combatTestTurnResponse("", "turn", 1),
|
||||
combatTestTurnResponse("mira thorn", "turn", 1),
|
||||
combatTestTurnResponse("Mira Thorn", "reaction", 3),
|
||||
combatTestTurnResponse("Hooded Guard", "turn", 5),
|
||||
}}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
fingerprints := prepared.CheckpointFingerprints()
|
||||
assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":npc_registry")
|
||||
assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":scene_eligibility")
|
||||
assertCombatFingerprint(t, fingerprints, "normalize:combat:"+combatnormalize.Key+":npc_registry")
|
||||
if len(fingerprints) == 0 {
|
||||
t.Fatal("checkpoint fingerprints = empty, want production combat identities")
|
||||
}
|
||||
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
@@ -74,20 +66,14 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if len(client.requests) != 4 || len(output.Rejected) != 0 {
|
||||
t.Fatalf("LLM requests = %d rejected = %#v, want one retry and no durable rejection", len(client.requests), output.Rejected)
|
||||
if len(client.requests) != 4 || len(output.OutputFiles) != 1 || output.FallbackCount != 0 {
|
||||
t.Fatalf("native calls=%d output=%#v, want corrected accepted combat", len(client.requests), output)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 || len(output.OutputFiles) == 0 {
|
||||
t.Fatalf("manifest/output = %#v / %#v, want approved combat JSON output", output.Manifest, output.OutputFiles)
|
||||
}
|
||||
serialized := output.NormalizeOutputs[0]
|
||||
if serialized.LaneID != "combat" || serialized.NormalizerKey != combatnormalize.Key || serialized.Artifact.Schema.ID != combatcodec.SchemaID {
|
||||
t.Fatalf("serialized combat output = %#v, want production combat lane schema", serialized)
|
||||
}
|
||||
value, err := combatcodec.New().Decode(serialized.Artifact.Content)
|
||||
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(combat output) error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(value.CombatTurns) != 3 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[1].Actor != "Mira Thorn" || value.CombatTurns[2].Actor != "Hooded Guard" {
|
||||
t.Fatalf("normalized combat output = %#v, want ordered canonical actors", value)
|
||||
}
|
||||
@@ -101,21 +87,6 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
if !hasCombatWarning(output.Diagnostics, combatnormalize.ReasonCodeActorCanonicalized) {
|
||||
t.Fatalf("diagnostics = %#v, want registry normalization advisory", output.Diagnostics)
|
||||
}
|
||||
if len(output.Manifest.References) != 3 {
|
||||
t.Fatalf("manifest references = %#v, want scene eligibility plus extract and normalize NPC provenance", output.Manifest.References)
|
||||
}
|
||||
lane := output.Manifest.ArtifactLanes[0]
|
||||
if lane.ID != "combat" || lane.Extractor != combatextract.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != combatnormalize.Key {
|
||||
t.Fatalf("manifest lane = %#v, want complete combat composition", lane)
|
||||
}
|
||||
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok || extractorMetadata["npc_count"] != 2 || extractorMetadata["npc_registry_digest"] == "" || extractorMetadata["scene_description_count"] != 3 || extractorMetadata["scene_eligibility_digest"] == "" || extractorMetadata["scene_gate_policy"] != "dnd.combat_turns.scene_gate.v1" {
|
||||
t.Fatalf("extractor metadata = %#v, want registry and scene eligibility metadata", lane.Metadata)
|
||||
}
|
||||
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
||||
if !ok || normalizerMetadata["npc_count"] != 2 || normalizerMetadata["normalization_policy"] != combatnormalize.NormalizationPolicy {
|
||||
t.Fatalf("normalizer metadata = %#v, want registry and normalization policy", lane.Metadata)
|
||||
}
|
||||
var combatFile *contracts.OutputFile
|
||||
for index := range output.OutputFiles {
|
||||
if output.OutputFiles[index].Name == "lanes/combat.json" {
|
||||
@@ -128,7 +99,7 @@ func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidation(t *testing.T) {
|
||||
func TestProductionCombatPipelineStopsAfterNativeStructuralFailure(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := combatOnlyConfig()
|
||||
profile := configValue.Pipelines["dnd-combat-fixture"]
|
||||
@@ -148,17 +119,12 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
client := &fakeCombatLLMClient{responses: []string{
|
||||
combatTestInvalidEnumResponse("unsupported"),
|
||||
combatTestInvalidEnumResponse("invalid"),
|
||||
combatTestInvalidEnumResponse("unknown"),
|
||||
}}
|
||||
client := &fakeCombatLLMClient{response: combatTestInvalidEnumResponse("unsupported")}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, moduleCatalog(registries), pipeline.ReferenceMaterializationOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v", err)
|
||||
}
|
||||
materialized.Steps[0].ArtifactLanes[0].ExtractValidationPolicy.SemanticRejection = pipeline.SemanticRejectionRejectOutput
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
@@ -167,18 +133,12 @@ func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidatio
|
||||
RawInput: readNPCFixture(t),
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
||||
|
||||
if !errors.Is(err, promptkit.ErrValidation) {
|
||||
t.Fatalf("Run() error=%v, want native structural failure", err)
|
||||
}
|
||||
if len(client.requests) != 3 || len(output.Rejected) != 1 {
|
||||
t.Fatalf("LLM requests = %d rejected = %#v, want exhausted retry and one rejection", len(client.requests), output.Rejected)
|
||||
}
|
||||
rejected := output.Rejected[0]
|
||||
if rejected.ReasonCode != combatshape.ReasonCode || rejected.ValidatorName != combatshape.Key || rejected.AttemptCount != 3 {
|
||||
t.Fatalf("rejected output = %#v, want exhausted combat shape rejection", rejected)
|
||||
}
|
||||
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("output = %#v, want non-fatal rejected combat result without normalized artifacts", output)
|
||||
if len(client.requests) == 0 || len(output.OutputFiles) != 0 {
|
||||
t.Fatalf("native calls=%d output=%#v, want exhausted rejection", len(client.requests), output)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,7 +165,7 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t
|
||||
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
||||
}
|
||||
client := &fakeCombatLLMClient{}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
@@ -217,13 +177,10 @@ func TestCombatPipelinePublishesUnavailableSceneWarningForMismatchedReference(t
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(client.requests) != 0 || len(output.Rejected) != 0 || !hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
|
||||
t.Fatalf("requests/rejected/diagnostics = %#v / %#v / %#v, want accepted unavailable-scene skip without a combat request", client.requests, output.Rejected, output.Diagnostics)
|
||||
if len(client.requests) != 0 || !hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
|
||||
t.Fatalf("native calls=%d diagnostics=%#v, want unavailable-scene skip", len(client.requests), output.Diagnostics)
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("normalized outputs = %#v, want accepted empty combat lane", output.NormalizeOutputs)
|
||||
}
|
||||
value, err := combatcodec.New().Decode(output.NormalizeOutputs[0].Artifact.Content)
|
||||
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
|
||||
if err != nil {
|
||||
t.Fatalf("Decode(combat output) error = %v", err)
|
||||
}
|
||||
@@ -297,7 +254,7 @@ func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecutio
|
||||
t.Fatalf("MaterializeReferences() error = %v, want materialization before typed preparation", err)
|
||||
}
|
||||
client := &fakeCombatLLMClient{}
|
||||
if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}); err == nil {
|
||||
if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)}); err == nil {
|
||||
t.Fatal("Prepare() error = nil, want NPC registry preparation failure")
|
||||
}
|
||||
if len(client.requests) != 0 {
|
||||
@@ -352,7 +309,7 @@ func combatOnlyConfig() config.Config {
|
||||
References: map[string]pipeline.ReferenceSource{"scene_descriptions": pipeline.ExternalReference(repositoryPathForIntegration("internal", "modules", "integration", "testdata", "combat_scene_descriptions.json"))},
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"combat": {
|
||||
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
|
||||
Extract: pipeline.ModuleBinding{Module: combatextract.Key},
|
||||
Normalize: pipeline.Binding(combatnormalize.Key),
|
||||
},
|
||||
},
|
||||
@@ -368,30 +325,26 @@ func repositoryPathForIntegration(parts ...string) string {
|
||||
|
||||
type fakeCombatLLMClient struct {
|
||||
mu sync.Mutex
|
||||
response string
|
||||
responses []string
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
}
|
||||
|
||||
func (client *fakeCombatLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (client *fakeCombatLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
snapshot, err := contracts.CloneStructuredCompletionRequest(req)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("clone fake request: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, snapshot)
|
||||
client.requests = append(client.requests, req)
|
||||
index := len(client.requests) - 1
|
||||
client.mu.Unlock()
|
||||
if client.response != "" {
|
||||
return &promptkit.GenerateResponse{Content: client.response}, nil
|
||||
}
|
||||
if index >= len(client.responses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("no combat fake response for call %d", index)
|
||||
return nil, fmt.Errorf("no combat fake response for call %d", index)
|
||||
}
|
||||
content := []byte(client.responses[index])
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate combat fake target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "combat-fake"}, nil
|
||||
return &promptkit.GenerateResponse{Content: client.responses[index]}, nil
|
||||
}
|
||||
|
||||
func combatTestInvalidEnumResponse(turnKind string) string {
|
||||
@@ -411,14 +364,4 @@ func hasCombatWarning(diagnostics contracts.DiagnosticCollection, reason string)
|
||||
return false
|
||||
}
|
||||
|
||||
func assertCombatFingerprint(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) {
|
||||
t.Helper()
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && strings.HasPrefix(fingerprint.Value, "sha256:") {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("fingerprints = %#v, want %q with semantic digest", fingerprints, name)
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*fakeCombatLLMClient)(nil)
|
||||
var _ promptkit.LLMClient = (*fakeCombatLLMClient)(nil)
|
||||
|
||||
@@ -2,7 +2,6 @@ package integration_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -14,21 +13,14 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/evidencecontext"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/semanticreconcile"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
||||
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcregistry"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/npcregistry"
|
||||
sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
npcnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/npcregistry"
|
||||
combatsemantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T) {
|
||||
@@ -38,233 +30,93 @@ func TestNPCOutputGroundsSpellAndCombatConsumersThroughOneOperation(t *testing.T
|
||||
campaignInputs := configureGroundedCampaignReferences(t, &configValue)
|
||||
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
||||
if err != nil || len(warnings) != 0 {
|
||||
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
||||
t.Fatalf("materialize references: %v warnings=%#v", err, warnings)
|
||||
}
|
||||
|
||||
client := &groundedDNDLLMClient{}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
for name, value := range map[string]string{
|
||||
"extract:npc_registry:dnd/npc-registry:mapping_policy": "dnd.npc_registry.extract_mapping.v3",
|
||||
"normalize:npc_registry:dnd/npc-registry:identity_policy": "dnd.npc_registry.identity.v1",
|
||||
"normalize:npc_registry:dnd/npc-registry:normalization_policy": npcnormalize.NormalizationPolicy,
|
||||
"normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_policy": semanticreconcile.Policy,
|
||||
"extract:spells:dnd/spells:mapping_policy": "dnd.spells.extract_mapping.v3",
|
||||
"extract:combat:dnd/combat-turns:scene_gate_policy": "dnd.combat_turns.scene_gate.v1",
|
||||
} {
|
||||
assertFingerprintValue(t, prepared.CheckpointFingerprints(), name, value)
|
||||
}
|
||||
for _, name := range []string{
|
||||
"extract:npc_registry:dnd/npc-registry:prompt",
|
||||
"extract:npc_registry:dnd/npc-registry:response_schema",
|
||||
"normalize:npc_registry:dnd/npc-registry:prompt",
|
||||
"normalize:npc_registry:dnd/npc-registry:response_schema",
|
||||
"normalize:npc_registry:dnd/npc-registry:semantic_reconciliation_limits",
|
||||
"extract:spells:dnd/spells:prompt",
|
||||
"extract:spells:dnd/spells:response_schema",
|
||||
"extract:spells:dnd/spells:npc_registry",
|
||||
"extract:combat:dnd/combat-turns:prompt",
|
||||
"extract:combat:dnd/combat-turns:response_schema",
|
||||
"extract:combat:dnd/combat-turns:scene_eligibility",
|
||||
"extract:combat:dnd/combat-turns:npc_registry",
|
||||
"normalize:combat:dnd/combat-turns:npc_registry",
|
||||
} {
|
||||
assertCombatFingerprint(t, prepared.CheckpointFingerprints(), name)
|
||||
}
|
||||
checkpoint := newGeneratedReferenceCheckpointLoader()
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readNPCFixture(t),
|
||||
Workers: 1,
|
||||
Checkpoint: checkpoint,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
}
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 4 {
|
||||
t.Fatalf("run outputs = %#v rejected = %#v, want NPC, scene, spell, and combat outputs", output.NormalizeOutputs, output.Rejected)
|
||||
}
|
||||
wantSchemas := map[string]string{"npc_registry": npccodec.SchemaID, "scene-descriptions": scenecodec.SchemaID, "spells": spellcodec.SchemaID, "combat": combatcodec.SchemaID}
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.Artifact.Schema.ID != wantSchemas[serialized.LaneID] || serialized.Artifact.Schema.Version != "v1" {
|
||||
t.Fatalf("%s artifact schema = %#v, want minimal v1 identity", serialized.LaneID, serialized.Artifact.Schema)
|
||||
}
|
||||
}
|
||||
wantExtractorIdentity := map[string]struct{ promptID, schemaID string }{
|
||||
"npc_registry": {npcregistry.PromptID, npcregistry.ResponseSchemaID},
|
||||
"scene-descriptions": {sceneextract.PromptID, sceneextract.ResponseSchemaID},
|
||||
"spells": {spells.PromptID, spells.ResponseSchemaID},
|
||||
"combat": {combatextract.PromptID, combatextract.ResponseSchemaID},
|
||||
}
|
||||
for _, lane := range output.Manifest.ArtifactLanes {
|
||||
want, ok := wantExtractorIdentity[lane.ID]
|
||||
metadata, metadataOK := lane.Metadata["extractor"].(map[string]any)
|
||||
if !ok || !metadataOK || metadata["prompt_id"] != want.promptID || metadata["prompt_version"] != "v1" || metadata["response_schema_id"] != want.schemaID || metadata["response_schema_version"] != "v1" {
|
||||
t.Fatalf("%s extractor metadata = %#v, want v1 prompt/schema identity", lane.ID, metadata)
|
||||
}
|
||||
}
|
||||
|
||||
var npcPayload, scenePayload []byte
|
||||
seenSteps := map[string]string{}
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
seenSteps[serialized.LaneID] = serialized.StepID
|
||||
switch serialized.LaneID {
|
||||
case "npc_registry":
|
||||
npcPayload = append([]byte(nil), serialized.Artifact.Content...)
|
||||
case "scene-descriptions":
|
||||
scenePayload = append([]byte(nil), serialized.Artifact.Content...)
|
||||
}
|
||||
}
|
||||
if seenSteps["npc_registry"] != "identify-npcs" || seenSteps["scene-descriptions"] != "identify-npcs" || seenSteps["spells"] != "grounded-events" || seenSteps["combat"] != "grounded-events" {
|
||||
t.Fatalf("normalized output steps = %#v, want ordered producer and consumer steps", seenSteps)
|
||||
}
|
||||
if len(npcPayload) == 0 {
|
||||
t.Fatal("NPC producer did not publish a canonical payload")
|
||||
}
|
||||
if len(scenePayload) == 0 {
|
||||
t.Fatal("scene producer did not publish a canonical payload")
|
||||
}
|
||||
npcValue, err := npccodec.New().Decode(npcPayload)
|
||||
if err != nil {
|
||||
t.Fatalf("decode NPC producer payload: %v", err)
|
||||
}
|
||||
npcPayload, err = npccodec.New().Encode(npcValue)
|
||||
if err != nil {
|
||||
t.Fatalf("encode canonical NPC producer payload: %v", err)
|
||||
}
|
||||
canonicalDigest := digestBytes(npcPayload)
|
||||
sceneValue, err := scenecodec.New().Decode(scenePayload)
|
||||
if err != nil {
|
||||
t.Fatalf("decode scene producer payload: %v", err)
|
||||
}
|
||||
scenePayload, err = scenecodec.New().Encode(sceneValue)
|
||||
if err != nil {
|
||||
t.Fatalf("encode canonical scene producer payload: %v", err)
|
||||
}
|
||||
sceneDigest := digestBytes(scenePayload)
|
||||
projection := []byte(`{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`)
|
||||
projectionDigest := digestBytes(projection)
|
||||
seenConsumers := map[string]bool{}
|
||||
for _, request := range client.requestsSnapshot() {
|
||||
if request.PromptID != spells.PromptID && request.PromptID != combatextract.PromptID {
|
||||
continue
|
||||
}
|
||||
input := request.Inputs["npc_registry"]
|
||||
if input.MediaType != npccodec.MediaType || input.Digest != projectionDigest || string(input.Content) != string(projection) || input.OriginURI != "" {
|
||||
t.Fatalf("%s NPC prompt input = %#v, want names-only generated registry", request.PromptID, input)
|
||||
}
|
||||
if request.PromptID == combatextract.PromptID {
|
||||
transcript, ok := request.Inputs["transcript"]
|
||||
if !ok || !strings.Contains(string(transcript.Content), "Mira Thorn asks the party") {
|
||||
t.Fatalf("combat transcript input = %#v, want configured transcript content", transcript)
|
||||
}
|
||||
for name, want := range campaignInputs {
|
||||
input, ok := request.Inputs[name]
|
||||
if !ok || string(input.Content) != want {
|
||||
t.Fatalf("combat %s input = %#v, want configured campaign reference", name, input)
|
||||
}
|
||||
}
|
||||
if _, ok := request.Inputs["scene_descriptions"]; ok {
|
||||
t.Fatalf("combat prompt inputs = %#v, must not contain scene descriptions", request.Inputs)
|
||||
}
|
||||
}
|
||||
seenConsumers[request.PromptID] = true
|
||||
}
|
||||
if !seenConsumers[spells.PromptID] || !seenConsumers[combatextract.PromptID] {
|
||||
t.Fatalf("consumer prompt IDs = %#v, want spell and combat requests", seenConsumers)
|
||||
}
|
||||
if combatCompletionCount(client.requestsSnapshot()) != 1 {
|
||||
t.Fatalf("combat completion requests = %#v, want exactly one for the exact combat scene", client.requestsSnapshot())
|
||||
}
|
||||
|
||||
provenanceCount := 0
|
||||
for _, reference := range output.Manifest.References {
|
||||
if reference.SlotName != "npc_registry" {
|
||||
continue
|
||||
}
|
||||
provenanceCount++
|
||||
if reference.Digest != canonicalDigest || reference.OriginType != "generated" {
|
||||
t.Fatalf("NPC generated provenance = %#v, want canonical digest and generated origin", reference)
|
||||
}
|
||||
}
|
||||
if provenanceCount != 3 {
|
||||
t.Fatalf("NPC generated provenance count = %d, want spell extract plus combat extract/normalize", provenanceCount)
|
||||
}
|
||||
sceneProvenanceCount := 0
|
||||
for _, reference := range output.Manifest.References {
|
||||
if reference.SlotName == "scene_descriptions" {
|
||||
sceneProvenanceCount++
|
||||
if reference.Stage != string(pipeline.StageExtract) || reference.StepID != "grounded-events" || reference.LaneID != "combat" || reference.OriginType != "generated" || reference.Digest != sceneDigest || reference.ArtifactKind != string(dnd.SceneDescriptionListKind) || reference.SchemaID != scenecodec.SchemaID || reference.SchemaName != scenecodec.SchemaName || reference.SchemaVersion != scenecodec.SchemaVersion || reference.MediaType != scenecodec.MediaType || reference.ProducerPipeline != "dnd-npc-grounded" || reference.ProducerStep != "identify-npcs" || reference.ProducerLane != "scene-descriptions" || reference.ProducerModule != "dnd/scene-descriptions" {
|
||||
t.Fatalf("scene generated provenance = %#v, want canonical combat handoff identity", reference)
|
||||
}
|
||||
}
|
||||
}
|
||||
if sceneProvenanceCount != 1 {
|
||||
t.Fatalf("scene generated provenance count = %d, want combat extractor handoff", sceneProvenanceCount)
|
||||
}
|
||||
assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
|
||||
manifestContent, err := json.Marshal(output.Manifest)
|
||||
client := &groundedDNDLLMClient{sceneTitle: "eligibility-only-scene-sentinel"}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"Mira Thorn", "Hooded Guard", "For every NPC record", "Extract Dungeons & Dragons", "A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
|
||||
if strings.Contains(string(manifestContent), forbidden) {
|
||||
t.Fatalf("manifest exposes prompt or reference payload content %q", forbidden)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{Prepared: prepared, RawInput: readNPCFixture(t), Workers: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, lane := range output.Manifest.ArtifactLanes {
|
||||
componentContent, err := json.Marshal(lane.Metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal %s component metadata: %v", lane.ID, err)
|
||||
}
|
||||
for _, forbidden := range []string{"A combat encounter", "The party faces an active encounter.", "Mira Thorn asks the party", `"scenes":`} {
|
||||
if strings.Contains(string(componentContent), forbidden) {
|
||||
t.Fatalf("%s component metadata exposes scene or transcript content %q", lane.ID, forbidden)
|
||||
}
|
||||
}
|
||||
for _, component := range []string{"extractor", "normalizer"} {
|
||||
metadata, ok := lane.Metadata[component].(map[string]any)
|
||||
if ok && (metadata["npc_registry_digest"] != nil || metadata["scene_eligibility_digest"] != nil) {
|
||||
t.Fatalf("%s %s metadata = %#v, want generated identity only in framework provenance", lane.ID, component, metadata)
|
||||
}
|
||||
}
|
||||
if len(output.OutputFiles) != 4 || output.FallbackCount != 0 {
|
||||
t.Fatalf("output=%#v, want four accepted domain artifacts", output)
|
||||
}
|
||||
|
||||
var combatValue dnd.CombatTurnList
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
switch serialized.LaneID {
|
||||
case "spells":
|
||||
spellValue := decodeRunnerSpellResponse(t, serialized.Artifact.Content)
|
||||
if len(spellValue.SpellCasts) != 1 || spellValue.SpellCasts[0].Caster != "Mira Thorn" {
|
||||
t.Fatalf("spell output = %#v, want one registry-grounded-context spell", spellValue)
|
||||
}
|
||||
assertCurrentEvidence(t, spellValue.SpellCasts[0].SourceRefs)
|
||||
case "combat":
|
||||
decoded, decodeErr := combatcodec.New().Decode(serialized.Artifact.Content)
|
||||
if decodeErr != nil {
|
||||
t.Fatalf("decode combat output: %v", decodeErr)
|
||||
}
|
||||
combatValue = decoded
|
||||
}
|
||||
npcs, err := npccodec.New().Decode(publishedLane(t, output, "npc_registry").Bytes)
|
||||
if err != nil || len(npcs.NPCs) != 2 {
|
||||
t.Fatalf("NPC registry=%#v error=%v", npcs, err)
|
||||
}
|
||||
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
|
||||
if err != nil || len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||
t.Fatalf("scene descriptions=%#v error=%v", scenes, err)
|
||||
}
|
||||
|
||||
projection := `{"npcs":[{"name":"Mira Thorn"},{"name":"Hooded Guard"}]}`
|
||||
seenConsumers := map[string]bool{}
|
||||
for _, request := range client.requestsSnapshot() {
|
||||
kind := "spell_casts"
|
||||
if nativeSchemaHasProperty(request, "combat_turns") {
|
||||
kind = "combat_turns"
|
||||
} else if !nativeSchemaHasProperty(request, "spell_casts") {
|
||||
continue
|
||||
}
|
||||
input, err := nativeJSONInput(request, "npcs")
|
||||
if err != nil || string(input) != projection {
|
||||
t.Fatalf("%s NPC projection=%s error=%v", kind, input, err)
|
||||
}
|
||||
text := nativePromptText(request)
|
||||
for _, npc := range npcs.NPCs {
|
||||
if strings.Contains(text, npc.ID) {
|
||||
t.Fatalf("%s prompt contains opaque registry identity", kind)
|
||||
}
|
||||
}
|
||||
if kind == "combat_turns" {
|
||||
transcript, err := nativeJSONInput(request, "units")
|
||||
if err != nil || !strings.Contains(string(transcript), "Mira Thorn asks the party") {
|
||||
t.Fatalf("combat transcript=%s error=%v", transcript, err)
|
||||
}
|
||||
for name, want := range campaignInputs {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("combat prompt omitted %s reference", name)
|
||||
}
|
||||
}
|
||||
for _, excluded := range []string{client.sceneTitle, "The party faces an active encounter."} {
|
||||
if strings.Contains(text, excluded) {
|
||||
t.Fatalf("combat prompt included eligibility-only scene prose %q", excluded)
|
||||
}
|
||||
}
|
||||
}
|
||||
seenConsumers[kind] = true
|
||||
}
|
||||
if !seenConsumers["spell_casts"] || !seenConsumers["combat_turns"] || combatCompletionCount(client.requestsSnapshot()) != 1 {
|
||||
t.Fatalf("consumer native requests=%#v", seenConsumers)
|
||||
}
|
||||
spellsValue := decodeRunnerSpellResponse(t, publishedLane(t, output, "spells").Bytes)
|
||||
if len(spellsValue.SpellCasts) != 1 || spellsValue.SpellCasts[0].Caster != "Mira Thorn" {
|
||||
t.Fatalf("spells=%#v", spellsValue)
|
||||
}
|
||||
assertCurrentEvidence(t, spellsValue.SpellCasts[0].SourceRefs)
|
||||
combatValue := publishedCombatOutput(t, output)
|
||||
if len(combatValue.CombatTurns) != 1 || combatValue.CombatTurns[0].Actor != "Mira Thorn" {
|
||||
t.Fatalf("combat output = %#v, want registry-normalized actor", combatValue)
|
||||
t.Fatalf("combat=%#v", combatValue)
|
||||
}
|
||||
assertCurrentEvidence(t, combatValue.CombatTurns[0].SourceRefs)
|
||||
}
|
||||
|
||||
func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||
func TestSceneCombatSemanticsCorrectionRevalidatesRevisedScene(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||
sceneLane := profile.Steps[0].Artifacts["scene-descriptions"]
|
||||
sceneLane.Extract.Retries = 1
|
||||
turns := 1
|
||||
sceneLane.Extract.MaxAdditionalTurns = &turns
|
||||
sceneLane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("extract/dnd/scene-descriptions/shape"),
|
||||
@@ -277,48 +129,48 @@ func TestSceneCombatSemanticsCorrectionRetriesAndRevalidates(t *testing.T) {
|
||||
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||
|
||||
client := &groundedDNDLLMClient{sceneKinds: []dnd.SceneKind{dnd.SceneKindNarrative, dnd.SceneKindCombat}, combatSemanticsClassifications: []string{"combat", "combat"}}
|
||||
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("rejected = %#v, want corrected acceptance", output.Rejected)
|
||||
output := runGroundedPipeline(t, configValue, registries, client)
|
||||
if output.FallbackCount != 0 {
|
||||
t.Fatalf("unexpected fallback: %#v", output)
|
||||
}
|
||||
for _, group := range output.Diagnostics.Groups {
|
||||
if group.Origin.ValidatorKey == combatsemantics.Key {
|
||||
t.Fatalf("combat-semantics validator left a diagnostic after correction: %#v", group)
|
||||
}
|
||||
}
|
||||
var scenes dnd.SceneDescriptionList
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.LaneID == "scene-descriptions" {
|
||||
var err error
|
||||
scenes, err = scenecodec.New().Decode(serialized.Artifact.Content)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
scenes, err := scenecodec.New().Decode(publishedLane(t, output, "scene-descriptions").Bytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(scenes.Scenes) != 1 || scenes.Scenes[0].Kind != dnd.SceneKindCombat {
|
||||
t.Fatalf("accepted scenes = %#v, want corrected combat scene", scenes)
|
||||
}
|
||||
requests := client.requestsSnapshot()
|
||||
var sceneRequests, validatorRequests []contracts.StructuredCompletionRequest
|
||||
var sceneRequests, validatorRequests []promptkit.GenerateRequest
|
||||
for _, request := range requests {
|
||||
switch request.PromptID {
|
||||
case sceneextract.PromptID:
|
||||
switch {
|
||||
case nativeSchemaHasProperty(request, "kind"):
|
||||
sceneRequests = append(sceneRequests, request)
|
||||
case combatsemantics.PromptID:
|
||||
case nativeSchemaHasProperty(request, "classification"):
|
||||
validatorRequests = append(validatorRequests, request)
|
||||
}
|
||||
}
|
||||
if len(sceneRequests) != 2 || len(validatorRequests) != 2 || sceneRequests[0].Correction != nil || sceneRequests[1].Correction == nil {
|
||||
t.Fatalf("scene/validator requests = %d/%d; want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||
if len(sceneRequests) != 2 || len(validatorRequests) != 2 {
|
||||
t.Fatalf("scene/validator calls=%d/%d, want correction and revalidation", len(sceneRequests), len(validatorRequests))
|
||||
}
|
||||
correction := sceneRequests[1].Correction
|
||||
if !strings.Contains(string(correction.AssistantResponse), `"kind":"narrative"`) || !strings.Contains(correction.UserGuidance, "Return kind: combat") || !strings.Contains(correction.UserGuidance, "active encounter") {
|
||||
t.Fatalf("correction = %#v, want latest candidate and useful guidance", correction)
|
||||
original := sceneRequests[0].Prompt.Messages
|
||||
revised := sceneRequests[1].Prompt.Messages
|
||||
if len(revised) != len(original)+2 || !reflect.DeepEqual(revised[:len(original)], original) {
|
||||
t.Fatal("scene correction lost native conversation history")
|
||||
}
|
||||
assistant, guidance := revised[len(original)], revised[len(original)+1]
|
||||
if assistant.Role != promptkit.RoleAssistant || !strings.Contains(assistant.Content, `"kind":"narrative"`) || guidance.Role != promptkit.RoleUser || !strings.Contains(guidance.Content, "Return kind: combat") || !strings.Contains(guidance.Content, "active encounter") {
|
||||
t.Fatalf("native correction messages=%#v", revised[len(original):])
|
||||
}
|
||||
for _, forbidden := range []string{combatsemantics.Key, combatsemantics.ReasonCodeActiveCombatNotClassified, "session:chunk"} {
|
||||
if strings.Contains(correction.UserGuidance, forbidden) {
|
||||
t.Fatalf("correction guidance leaked internal detail %q: %q", forbidden, correction.UserGuidance)
|
||||
if strings.Contains(guidance.Content, forbidden) {
|
||||
t.Fatalf("correction guidance contains diagnostic identifier %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -338,7 +190,7 @@ func TestProductionDNDOutputPublishesSelectedEvidenceContext(t *testing.T) {
|
||||
`"id": 4`, `"id": 50`,
|
||||
`"id": 5`, `"id": 40`,
|
||||
).Replace(string(readNPCFixture(t)))
|
||||
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, nil, []byte(raw))
|
||||
output := runGroundedPipelineWithRaw(t, configValue, registries, &groundedDNDLLMClient{firstUnitID: 10, thirdUnitID: 20}, []byte(raw))
|
||||
|
||||
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
||||
if err != nil {
|
||||
@@ -357,7 +209,7 @@ func TestProductionDNDOutputCanExplicitlySelectSceneDescriptionEvidence(t *testi
|
||||
"enabled": true, "lanes": []any{"scene-descriptions"},
|
||||
}}
|
||||
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{}, nil)
|
||||
output := runGroundedPipeline(t, configValue, registries, &groundedDNDLLMClient{})
|
||||
|
||||
value, err := evidencecontext.New().Decode(outputFileContent(t, output.OutputFiles, "evidence-context.json"))
|
||||
if err != nil {
|
||||
@@ -379,45 +231,20 @@ func evidenceUnitIDs(value evidencecontext.Document) []int {
|
||||
func TestGroundedPipelineSkipsCombatForExactNarrativeScene(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
configValue := loadGroundedPipelineConfig(t)
|
||||
profile := configValue.Pipelines["dnd-npc-grounded"]
|
||||
combat := profile.Steps[1].Artifacts["combat"]
|
||||
combat.Extract.Retries = 2
|
||||
profile.Steps[1].Artifacts["combat"] = combat
|
||||
configValue.Pipelines["dnd-npc-grounded"] = profile
|
||||
|
||||
client := &groundedDNDLLMClient{sceneKind: dnd.SceneKindNarrative}
|
||||
output := runGroundedPipeline(t, configValue, registries, client, nil)
|
||||
output := runGroundedPipeline(t, configValue, registries, client)
|
||||
if combatCompletionCount(client.requestsSnapshot()) != 0 {
|
||||
t.Fatalf("combat completion requests = %#v, want none for an exact narrative scene", client.requestsSnapshot())
|
||||
}
|
||||
if len(output.Rejected) != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
|
||||
t.Fatalf("narrative output rejected/diagnostics = %#v / %#v, want accepted skip without unavailable advisory", output.Rejected, output.Diagnostics)
|
||||
if output.FallbackCount != 0 || hasCombatWarning(output.Diagnostics, "scene_classification_unavailable") {
|
||||
t.Fatalf("narrative output=%#v, want accepted skip", output)
|
||||
}
|
||||
combatValue := normalizedCombatOutput(t, output)
|
||||
combatValue := publishedCombatOutput(t, output)
|
||||
if combatValue.CombatTurns == nil || len(combatValue.CombatTurns) != 0 {
|
||||
t.Fatalf("narrative combat output = %#v, want an accepted empty combat list", combatValue)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedSceneReferenceChangesCombatCheckpointDependency(t *testing.T) {
|
||||
registries := productionNPCRegistries(t)
|
||||
first := generatedSceneDependency(t, registries, "A combat encounter")
|
||||
second := generatedSceneDependency(t, registries, "A revised combat encounter")
|
||||
if first == second {
|
||||
t.Fatalf("generated combat dependencies = %q and %q, want canonical scene artifact change to invalidate reuse", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func assertFingerprintValue(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name, want string) {
|
||||
t.Helper()
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Name == name && fingerprint.Value == want {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("fingerprints = %#v, want %q = %q", fingerprints, name, want)
|
||||
}
|
||||
|
||||
func assertCurrentEvidence(t *testing.T, references []source.SourceRef) {
|
||||
t.Helper()
|
||||
for _, reference := range references {
|
||||
@@ -451,11 +278,11 @@ func configureGroundedCampaignReferences(t *testing.T, cfg *config.Config) map[s
|
||||
return values
|
||||
}
|
||||
|
||||
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader) pipeline.RunOutput {
|
||||
return runGroundedPipelineWithRaw(t, configValue, registries, client, checkpoint, readNPCFixture(t))
|
||||
func runGroundedPipeline(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient) pipeline.RunOutput {
|
||||
return runGroundedPipelineWithRaw(t, configValue, registries, client, readNPCFixture(t))
|
||||
}
|
||||
|
||||
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, checkpoint pipeline.CheckpointLoader, raw []byte) pipeline.RunOutput {
|
||||
func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registries pipeline.Registries, client *groundedDNDLLMClient, raw []byte) pipeline.RunOutput {
|
||||
t.Helper()
|
||||
catalog := moduleCatalog(registries)
|
||||
effective, err := configValue.Resolve(config.ResolveInput{PipelineID: "dnd-npc-grounded", Catalog: catalog})
|
||||
@@ -466,15 +293,14 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
|
||||
if err != nil || len(warnings) != 0 {
|
||||
t.Fatalf("MaterializeReferences() error = %v warnings = %#v", err, warnings)
|
||||
}
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
||||
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{Engine: nativeTestEngine(t, client)})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: append([]byte(nil), raw...),
|
||||
Workers: 1,
|
||||
Checkpoint: checkpoint,
|
||||
Prepared: prepared,
|
||||
RawInput: append([]byte(nil), raw...),
|
||||
Workers: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v", err)
|
||||
@@ -482,80 +308,28 @@ func runGroundedPipelineWithRaw(t *testing.T, configValue config.Config, registr
|
||||
return output
|
||||
}
|
||||
|
||||
func normalizedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
|
||||
func publishedCombatOutput(t *testing.T, output pipeline.RunOutput) dnd.CombatTurnList {
|
||||
t.Helper()
|
||||
for _, serialized := range output.NormalizeOutputs {
|
||||
if serialized.LaneID != "combat" {
|
||||
continue
|
||||
}
|
||||
value, err := combatcodec.New().Decode(serialized.Artifact.Content)
|
||||
if err != nil {
|
||||
t.Fatalf("decode combat output: %v", err)
|
||||
}
|
||||
return value
|
||||
value, err := combatcodec.New().Decode(publishedLane(t, output, "combat").Bytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Fatalf("normalized outputs = %#v, want combat lane", output.NormalizeOutputs)
|
||||
return dnd.CombatTurnList{}
|
||||
return value
|
||||
}
|
||||
|
||||
func combatCompletionCount(requests []contracts.StructuredCompletionRequest) int {
|
||||
func combatCompletionCount(requests []promptkit.GenerateRequest) int {
|
||||
count := 0
|
||||
for _, request := range requests {
|
||||
if request.PromptID == combatextract.PromptID {
|
||||
if nativeSchemaHasProperty(request, "combat_turns") {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func generatedSceneDependency(t *testing.T, registries pipeline.Registries, sceneTitle string) string {
|
||||
t.Helper()
|
||||
client := &groundedDNDLLMClient{sceneTitle: sceneTitle}
|
||||
checkpoint := newGeneratedReferenceCheckpointLoader()
|
||||
runGroundedPipeline(t, loadGroundedPipelineConfig(t), registries, client, checkpoint)
|
||||
return assertGeneratedSceneDependency(t, checkpoint.extractDependencies("combat"))
|
||||
}
|
||||
|
||||
func assertGeneratedSceneDependency(t *testing.T, dependencies []pipeline.CheckpointFingerprint) string {
|
||||
t.Helper()
|
||||
for _, dependency := range dependencies {
|
||||
if dependency.Name == "generated-reference:scene_descriptions:0" && strings.HasPrefix(dependency.Value, "sha256:") {
|
||||
return dependency.Value
|
||||
}
|
||||
}
|
||||
t.Fatalf("combat extract checkpoint dependencies = %#v, want generated scene reference fingerprint", dependencies)
|
||||
return ""
|
||||
}
|
||||
|
||||
type generatedReferenceCheckpointLoader struct {
|
||||
pipeline.CheckpointLoader
|
||||
extract map[string][][]pipeline.CheckpointFingerprint
|
||||
}
|
||||
|
||||
func newGeneratedReferenceCheckpointLoader() *generatedReferenceCheckpointLoader {
|
||||
return &generatedReferenceCheckpointLoader{
|
||||
CheckpointLoader: pipeline.NoopCheckpointLoader(),
|
||||
extract: make(map[string][][]pipeline.CheckpointFingerprint),
|
||||
}
|
||||
}
|
||||
|
||||
func (loader *generatedReferenceCheckpointLoader) Enabled() bool { return true }
|
||||
|
||||
func (loader *generatedReferenceCheckpointLoader) Extract(laneID, _ string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
||||
loader.extract[laneID] = append(loader.extract[laneID], append([]pipeline.CheckpointFingerprint(nil), dependencies...))
|
||||
return pipeline.ExtractCheckpoint{}, pipeline.NewCheckpointDecision(pipeline.CheckpointDecisionExecuted, pipeline.CheckpointReasonMissing)
|
||||
}
|
||||
|
||||
func (loader *generatedReferenceCheckpointLoader) extractDependencies(laneID string) []pipeline.CheckpointFingerprint {
|
||||
if len(loader.extract[laneID]) != 1 {
|
||||
return nil
|
||||
}
|
||||
return loader.extract[laneID][0]
|
||||
}
|
||||
|
||||
type groundedDNDLLMClient struct {
|
||||
mu sync.Mutex
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
requests []promptkit.GenerateRequest
|
||||
sceneKind dnd.SceneKind
|
||||
sceneTitle string
|
||||
firstUnitID int
|
||||
@@ -566,13 +340,13 @@ type groundedDNDLLMClient struct {
|
||||
combatSemanticsCalls int
|
||||
}
|
||||
|
||||
func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, request contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
func (client *groundedDNDLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
client.mu.Lock()
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(request))
|
||||
client.mu.Unlock()
|
||||
defer client.mu.Unlock()
|
||||
client.requests = append(client.requests, request)
|
||||
|
||||
var payload any
|
||||
firstUnitID := client.firstUnitID
|
||||
@@ -583,8 +357,8 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
if thirdUnitID == 0 {
|
||||
thirdUnitID = 3
|
||||
}
|
||||
switch request.PromptID {
|
||||
case npcregistry.PromptID:
|
||||
switch {
|
||||
case nativeSchemaHasProperty(request, "npcs"):
|
||||
payload = map[string]any{"npcs": []any{
|
||||
map[string]any{
|
||||
"name": "Mira Thorn", "source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
||||
@@ -593,9 +367,9 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
"name": "Hooded Guard", "source_refs": []any{map[string]int{"start_unit_id": thirdUnitID, "end_unit_id": thirdUnitID}},
|
||||
},
|
||||
}}
|
||||
case npcnormalize.PromptID:
|
||||
case nativeSchemaHasProperty(request, "duplicate_groups"):
|
||||
payload = map[string]any{"duplicate_groups": []any{}}
|
||||
case sceneextract.PromptID:
|
||||
case nativeSchemaHasProperty(request, "kind"):
|
||||
kind := client.sceneKind
|
||||
if client.sceneCalls < len(client.sceneKinds) {
|
||||
kind = client.sceneKinds[client.sceneCalls]
|
||||
@@ -609,46 +383,38 @@ func (client *groundedDNDLLMClient) CompleteStructured(ctx context.Context, requ
|
||||
title = "A combat encounter"
|
||||
}
|
||||
payload = map[string]any{"kind": kind, "title": title, "summary": "The party faces an active encounter."}
|
||||
case combatsemantics.PromptID:
|
||||
case nativeSchemaHasProperty(request, "classification"):
|
||||
classification := "combat"
|
||||
if client.combatSemanticsCalls < len(client.combatSemanticsClassifications) {
|
||||
classification = client.combatSemanticsClassifications[client.combatSemanticsCalls]
|
||||
}
|
||||
client.combatSemanticsCalls++
|
||||
payload = map[string]any{"classification": classification, "explanation": "The transcript shows an active encounter with hostile action."}
|
||||
case spells.PromptID:
|
||||
case nativeSchemaHasProperty(request, "spell_casts"):
|
||||
payload = map[string]any{"spell_casts": []any{map[string]any{
|
||||
"caster": "Mira Thorn", "spell": "Cure Wounds",
|
||||
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
||||
}}}
|
||||
case combatextract.PromptID:
|
||||
case nativeSchemaHasProperty(request, "combat_turns"):
|
||||
payload = map[string]any{"combat_turns": []any{map[string]any{
|
||||
"actor": "Mira Thorn",
|
||||
"turn_kind": "turn",
|
||||
"source_refs": []any{map[string]int{"start_unit_id": firstUnitID, "end_unit_id": firstUnitID}},
|
||||
}}}
|
||||
default:
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected grounded prompt %q", request.PromptID)
|
||||
return nil, fmt.Errorf("unexpected native grounded response contract")
|
||||
}
|
||||
content, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate grounded response: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "grounded-fake"}, nil
|
||||
return &promptkit.GenerateResponse{Content: string(content)}, nil
|
||||
}
|
||||
|
||||
func (client *groundedDNDLLMClient) requestsSnapshot() []contracts.StructuredCompletionRequest {
|
||||
func (client *groundedDNDLLMClient) requestsSnapshot() []promptkit.GenerateRequest {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
return append([]contracts.StructuredCompletionRequest(nil), client.requests...)
|
||||
return append([]promptkit.GenerateRequest(nil), client.requests...)
|
||||
}
|
||||
|
||||
func digestBytes(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return fmt.Sprintf("sha256:%x", sum[:])
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*groundedDNDLLMClient)(nil)
|
||||
var _ promptkit.LLMClient = (*groundedDNDLLMClient)(nil)
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
version: 5
|
||||
version: 6
|
||||
output:
|
||||
directory: ./notarius-output
|
||||
cache:
|
||||
chunk_plans:
|
||||
mode: bypass
|
||||
checkpoints: {}
|
||||
debug:
|
||||
directory: ./notarius-debug
|
||||
pipelines:
|
||||
|
||||
Reference in New Issue
Block a user