Migrate scene descriptions to native correction with invariant-preserving final fallbacks
This commit is contained in:
@@ -2,9 +2,12 @@ package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
@@ -13,6 +16,8 @@ import (
|
||||
)
|
||||
|
||||
const Key = "dnd/scene-descriptions"
|
||||
const PromptID = "notarius/dnd.scene_descriptions"
|
||||
const CorrectionPromptID = "notarius/dnd.scene_descriptions.repair"
|
||||
|
||||
var requiredCapabilities = []string{
|
||||
"chunks",
|
||||
@@ -41,102 +46,65 @@ func referenceSlots() []contracts.ReferenceSlot {
|
||||
}
|
||||
|
||||
var _ contracts.Extractor[dnd.SceneDescriptionList] = (*Extractor)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Extractor)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Extractor)(nil)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type Extractor struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
type Extractor struct{ engine *promptkit.Engine }
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if llmClient == nil {
|
||||
return nil, extractorErrorf("LLM client must not be nil")
|
||||
func New(engine *promptkit.Engine, _ Options, references ...contracts.ReferenceSet) (*Extractor, error) {
|
||||
if engine == nil {
|
||||
return nil, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
if len(references) > 1 {
|
||||
return nil, extractorErrorf("at most one reference set may be supplied")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, extractorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Extractor{
|
||||
llm: llmClient,
|
||||
promptSHA: promptSHA,
|
||||
responseSchemaSHA: responseSchema.SHA256,
|
||||
}, nil
|
||||
return &Extractor{engine: engine}, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) Key() string { return Key }
|
||||
|
||||
func (e *Extractor) ReferenceSlots() []contracts.ReferenceSlot { return referenceSlots() }
|
||||
|
||||
func (e *Extractor) ManifestMetadata() map[string]any {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": e.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": e.responseSchemaSHA,
|
||||
"mapping_policy": mappingPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: e.promptSHA},
|
||||
{Name: "response_schema", Value: e.responseSchemaSHA},
|
||||
{Name: "mapping_policy", Value: mappingPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) {
|
||||
func (e *Extractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) (contracts.TypedExtractionResult[dnd.SceneDescriptionList], error) {
|
||||
if e == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("extractor must not be nil")
|
||||
}
|
||||
if e.llm == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("LLM client must not be nil")
|
||||
if e.engine == nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("native engine must not be nil")
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, req)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("%w", err)
|
||||
}
|
||||
|
||||
var response extractionResponse
|
||||
completion, err := e.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Correction: req.Correction,
|
||||
Inputs: shared.PromptInputs(sourceInput, req.References),
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("complete structured output: %w", err)
|
||||
chunk := source.Chunk{ID: req.Chunk.ID, Ref: req.Chunk.Ref}
|
||||
index := source.NewDocumentIndex(req.Source)
|
||||
materials := shared.PromptInputs(sourceInput, req.References)
|
||||
inputs := make(map[string]promptkit.ArtifactRef, len(materials))
|
||||
for name, material := range materials {
|
||||
inputs[name] = promptkit.InlineWithURI(material.OriginURI, string(material.Content))
|
||||
}
|
||||
candidate, err := shared.ModelCandidateFromResponse(completion)
|
||||
initial := promptkit.RunRequest{PromptID: PromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID, Inputs: inputs}
|
||||
value, outcome, err := pipeline.RunProducer(ctx, e.engine, req.Operation, initial,
|
||||
func(_ context.Context, result *promptkit.RunResult) (*dnd.SceneDescriptionList, []string, error) {
|
||||
var response extractionResponse
|
||||
if err := json.Unmarshal([]byte(result.RawOutput), &response); err != nil {
|
||||
return nil, nil, extractorErrorf("decode native output: %w", err)
|
||||
}
|
||||
value := mapResponse(response, &chunk)
|
||||
return &value, nil, nil
|
||||
},
|
||||
func(_ context.Context, guidance []string) (promptkit.RunRequest, error) {
|
||||
return promptkit.RunRequest{PromptID: CorrectionPromptID, ProfileID: req.LLMProfile, SessionID: req.SessionID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{"guidance": promptkit.Inline(strings.Join(guidance, "\n\n"))}}, nil
|
||||
},
|
||||
func(_ context.Context, final *dnd.SceneDescriptionList) (dnd.SceneDescriptionList, error) {
|
||||
return sceneFallback(final, chunk, index)
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("capture model candidate: %w", err)
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{}, extractorErrorf("produce scene description: %w", err)
|
||||
}
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: mapResponse(response, req.Chunk), ModelCandidate: candidate}, nil
|
||||
return contracts.TypedExtractionResult[dnd.SceneDescriptionList]{Value: value, Outcome: outcome}, nil
|
||||
}
|
||||
|
||||
func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDescriptionList {
|
||||
@@ -151,14 +119,14 @@ func mapResponse(response extractionResponse, chunk *source.Chunk) dnd.SceneDesc
|
||||
|
||||
func ModuleSpec() pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
Key: Key,
|
||||
Stage: pipeline.StageExtract,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
Prompts: []string{PromptID, CorrectionPromptID},
|
||||
Requires: append([]string(nil), requiredCapabilities...),
|
||||
Provides: append([]string(nil), providedCapabilities...),
|
||||
ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
ReferenceSlots: referenceSlots(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,7 +136,7 @@ func Register(registry *pipeline.ExtractorRegistry) error {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options, request.References)
|
||||
return New(request.Dependencies.Engine, options, request.References)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -32,12 +32,11 @@ func TestExtractMapsExactlyOneSceneToTheChunk(t *testing.T) {
|
||||
t.Fatalf("LLM calls = %d, want 1", len(client.requests))
|
||||
}
|
||||
completed := client.requests[0]
|
||||
if completed.StageName != Key || completed.PromptID != PromptID || completed.PromptVersion != SchemaVersion || completed.ProfileID != "profile-scene-descriptions" || completed.SessionID != "session-123" {
|
||||
t.Fatalf("LLM request identity = %#v", completed)
|
||||
if completed.Prompt.SessionID != request.SessionID || result.Outcome.Disposition != contracts.ProducerAccepted {
|
||||
t.Fatal("native session/outcome lost")
|
||||
}
|
||||
transcript := completed.Inputs["transcript"]
|
||||
if transcript.Name != "transcript" || transcript.MediaType != "application/json" || transcript.Digest != "sha256:chunk" || transcript.OriginURI != "file:///session-alpha.json" || string(transcript.Content) != string(request.Chunk.Content) {
|
||||
t.Fatalf("transcript input = %#v, want chunk-scoped material", transcript)
|
||||
if string(renderedInput(t, completed, "transcript")) != string(request.Chunk.Content) {
|
||||
t.Fatal("chunk-scoped material changed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,12 +46,6 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
for _, slot := range []string{"players", "party", "glossary"} {
|
||||
if got := string(client.requests[0].Inputs[slot].Content); got != " " {
|
||||
t.Fatalf("empty %s input = %q, want explicit placeholder", slot, got)
|
||||
}
|
||||
}
|
||||
|
||||
req.References = contracts.ReferenceSet{Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"players": {Slot: contracts.ReferenceSlot{Name: "players"}, Items: []contracts.ReferenceItem{{SlotName: "players", Content: []byte("Dana: Mira")}}},
|
||||
"party": {Slot: contracts.ReferenceSlot{Name: "party"}, Items: []contracts.ReferenceItem{{SlotName: "party", Content: []byte("Mira: ranger")}}},
|
||||
@@ -61,23 +54,21 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) {
|
||||
if _, err := newExtractor(t, client).Extract(context.Background(), req); err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
inputs := client.requests[1].Inputs
|
||||
if string(inputs["players"].Content) != "Dana: Mira" || string(inputs["party"].Content) != "Mira: ranger" || string(inputs["glossary"].Content) != "Greencloak: local title" {
|
||||
t.Fatalf("reference inputs = %#v", inputs)
|
||||
var text string
|
||||
for _, message := range client.requests[1].Prompt.Messages {
|
||||
text += message.Content
|
||||
}
|
||||
if strings.Contains(string(inputs["transcript"].Content), "local title") {
|
||||
t.Fatal("transcript input contains reference content")
|
||||
for _, reference := range []string{"Dana: Mira", "Mira: ranger", "Greencloak: local title"} {
|
||||
if strings.Count(text, reference) != 1 {
|
||||
t.Fatalf("optional input not rendered once: %q", reference)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(renderedInput(t, client.requests[1], "transcript")), "local title") {
|
||||
t.Fatal("campaign context became evidence")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v", err)
|
||||
}
|
||||
if err := validateJSONSchema(t, map[string]any{"kind": "narrative", "title": " ", "summary": ""}, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("semantic candidate rejected by private schema: %v", err)
|
||||
}
|
||||
client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{
|
||||
Kind: dnd.SceneKindNarrative, Title: " ", Summary: "",
|
||||
}}
|
||||
@@ -98,11 +89,11 @@ func TestExtractValidatesRequestsAndSurfacesProviderFailures(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
extractor *Extractor
|
||||
req contracts.TypedExtractionRequest
|
||||
req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]
|
||||
want string
|
||||
}{
|
||||
{name: "nil extractor", extractor: nilExtractor, req: request, want: "extractor"},
|
||||
{name: "nil LLM client", extractor: &Extractor{}, req: request, want: "LLM client"},
|
||||
{name: "nil LLM client", extractor: &Extractor{}, req: request, want: "native engine"},
|
||||
{name: "wrapped preflight failure", extractor: extractor, req: mismatchedSourceInputRequest(request), want: "must match chunk"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
35
internal/modules/dnd/extract/scenedescriptions/fallback.go
Normal file
35
internal/modules/dnd/extract/scenedescriptions/fallback.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
)
|
||||
|
||||
// sceneFallback checks construction invariants, not the configured validator chain.
|
||||
func sceneFallback(final *dnd.SceneDescriptionList, chunk source.Chunk, index source.DocumentIndex) (dnd.SceneDescriptionList, error) {
|
||||
if final == nil {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("no usable final scene description is available")
|
||||
}
|
||||
if len(final.Scenes) != 1 {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("scene fallback requires exactly one final scene")
|
||||
}
|
||||
scene := final.Scenes[0]
|
||||
if scene.ID != chunk.ID || scene.SourceRef != chunk.Ref {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene does not match the current chunk identity and range")
|
||||
}
|
||||
if scene.ID != strings.TrimSpace(scene.ID) || scene.SourceRef.SourceID != strings.TrimSpace(scene.SourceRef.SourceID) ||
|
||||
scene.Title != strings.TrimSpace(scene.Title) || scene.Summary != strings.TrimSpace(scene.Summary) {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene fields must be trimmed")
|
||||
}
|
||||
if _, err := scenecodec.New().Encode(*final); err != nil {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene cannot be published: %w", err)
|
||||
}
|
||||
if err := index.ValidateRef(scene.SourceRef); err != nil {
|
||||
return dnd.SceneDescriptionList{}, fmt.Errorf("final scene source range is unusable: %w", err)
|
||||
}
|
||||
return dnd.SceneDescriptionList{Scenes: append([]dnd.SceneDescription(nil), final.Scenes...)}, nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestSceneFallbackRequiresOneUsableFinalSceneWithCurrentIdentity(t *testing.T) {
|
||||
doc := &source.SourceDocument{ID: "session", Units: []source.SourceUnit{{ID: 1}, {ID: 2}}}
|
||||
chunk := source.Chunk{ID: "chunk", Ref: source.SourceRef{SourceID: "session", StartUnitID: 1, EndUnitID: 2}}
|
||||
index := source.NewDocumentIndex(doc)
|
||||
valid := dnd.SceneDescription{ID: chunk.ID, SourceRef: chunk.Ref, Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
change func(*dnd.SceneDescriptionList, *source.Chunk)
|
||||
}{
|
||||
{"empty", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes = []dnd.SceneDescription{} }},
|
||||
{"many", func(value *dnd.SceneDescriptionList, _ *source.Chunk) {
|
||||
value.Scenes = append(value.Scenes, value.Scenes[0])
|
||||
}},
|
||||
{"blank title", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Title = "" }},
|
||||
{"blank summary", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = "" }},
|
||||
{"unsupported kind", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Kind = "unknown" }},
|
||||
{"wrong identity", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].ID = "another-chunk" }},
|
||||
{"partial range", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.EndUnitID = 1 }},
|
||||
{"wrong source", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].SourceRef.SourceID = "other" }},
|
||||
{"unknown endpoint", func(value *dnd.SceneDescriptionList, current *source.Chunk) {
|
||||
value.Scenes[0].SourceRef.EndUnitID = 99
|
||||
current.Ref = value.Scenes[0].SourceRef
|
||||
}},
|
||||
{"untrimmed identity", func(value *dnd.SceneDescriptionList, current *source.Chunk) {
|
||||
value.Scenes[0].ID = " chunk "
|
||||
current.ID = value.Scenes[0].ID
|
||||
}},
|
||||
{"untrimmed prose", func(value *dnd.SceneDescriptionList, _ *source.Chunk) { value.Scenes[0].Summary = " Untrimmed " }},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}}
|
||||
current := chunk
|
||||
test.change(&value, ¤t)
|
||||
if result, err := sceneFallback(&value, current, index); err == nil || result.Scenes != nil {
|
||||
t.Fatalf("unusable fallback=%#v %v", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if result, err := sceneFallback(nil, chunk, index); err == nil || result.Scenes != nil {
|
||||
t.Fatalf("missing final scene=%#v %v", result, err)
|
||||
}
|
||||
value := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{valid}}
|
||||
result, err := sceneFallback(&value, chunk, index)
|
||||
if err != nil || !reflect.DeepEqual(result, value) {
|
||||
t.Fatalf("usable final scene=%#v %v", result, err)
|
||||
}
|
||||
result.Scenes[0].Title = "changed"
|
||||
if value.Scenes[0] != valid {
|
||||
t.Fatal("fallback aliases the candidate or alters its classification/prose")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
|
||||
semantics "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/combat_semantics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
sourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_refs"
|
||||
relatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/source_relatedness"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const narrativeScene = `{"kind":"narrative","title":"The watchtower","summary":"The party faces the bandits."}`
|
||||
const combatScene = `{"kind":"combat","title":"Watchtower fight","summary":"Bandits attack and the party drives them back."}`
|
||||
const blankScene = `{"kind":"combat","title":"","summary":""}`
|
||||
const combatJudgment = `{"classification":"combat","explanation":"Bandits attack the party during the current scene."}`
|
||||
|
||||
func bindSceneAssessment(t *testing.T, extractor *Extractor, request *contracts.TypedExtractionRequest[dnd.SceneDescriptionList], withSemantic bool) *[]string {
|
||||
t.Helper()
|
||||
validators := []contracts.TypedValidator[dnd.SceneDescriptionList]{shape.New(shape.Options{}), sourcerefs.New(sourcerefs.Options{}), relatedness.New(relatedness.Options{})}
|
||||
if withSemantic {
|
||||
validator, err := semantics.New(extractor.engine, semantics.Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validators = append(validators, validator)
|
||||
}
|
||||
var calls []string
|
||||
request.Operation.Assess = func(ctx context.Context, value dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||
var assessment contracts.CandidateAssessment
|
||||
for _, validator := range validators {
|
||||
calls = append(calls, validator.Name())
|
||||
profile := request.LLMProfile
|
||||
if validator.Name() == semantics.Key {
|
||||
profile = "scene-validator-profile"
|
||||
}
|
||||
result, err := validator.Validate(ctx, contracts.TypedValidationRequest[dnd.SceneDescriptionList]{
|
||||
Stage: "extract", Source: request.Source, Chunk: request.Chunk, SourceInput: request.SourceInput,
|
||||
LLMProfile: profile, SessionID: request.SessionID, Execution: request.Operation.Execution, Value: value,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.CandidateAssessment{}, err
|
||||
}
|
||||
decision := contracts.AssessmentRejected
|
||||
if result.Approved {
|
||||
decision = contracts.AssessmentApproved
|
||||
}
|
||||
assessment.Validators = append(assessment.Validators, contracts.ValidatorAssessment{ValidatorName: validator.Name(), Decision: decision, Result: result})
|
||||
}
|
||||
return assessment, nil
|
||||
}
|
||||
return &calls
|
||||
}
|
||||
|
||||
func TestSceneCorrectionUsesRealNativeCombatJudgmentsAndPreservesSourceIdentity(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
request.Operation.ReuseEligible = false
|
||||
request.Operation.Execution.Execution = &promptkit.ExecutionTargetOverride{Model: "run-wide-scene-model"}
|
||||
var prepared []promptkit.PreparedRun
|
||||
request.Operation.Execution.OnPrepared = func(_ context.Context, value promptkit.PreparedRun) { prepared = append(prepared, value) }
|
||||
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, combatScene, combatJudgment}}
|
||||
extractor := newExtractor(t, client)
|
||||
calls := bindSceneAssessment(t, extractor, &request, true)
|
||||
result, err := extractor.Extract(context.Background(), request)
|
||||
chain := []string{shape.Key, sourcerefs.Key, relatedness.Key, semantics.Key}
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerAccepted || result.Outcome.ReuseEligible ||
|
||||
!reflect.DeepEqual(*calls, append(append([]string(nil), chain...), chain...)) || len(client.requests) != 4 || len(prepared) != 4 || len(result.Value.Scenes) != 1 {
|
||||
t.Fatalf("scene correction=%#v %v calls=%#v native=%d/%d", result, err, *calls, len(client.requests), len(prepared))
|
||||
}
|
||||
scene := result.Value.Scenes[0]
|
||||
if scene.ID != request.Chunk.ID || scene.SourceRef != request.Chunk.Ref || scene.Kind != dnd.SceneKindCombat || scene.Title != "Watchtower fight" {
|
||||
t.Fatalf("mapped scene=%#v", scene)
|
||||
}
|
||||
for i, call := range client.requests {
|
||||
id, profile := PromptID, request.LLMProfile
|
||||
if i == 2 {
|
||||
id = CorrectionPromptID
|
||||
}
|
||||
if i%2 == 1 {
|
||||
id, profile = semantics.PromptID, "scene-validator-profile"
|
||||
}
|
||||
if prepared[i].PromptID != id || prepared[i].SelectedProfileID != profile || call.Target.Model != "run-wide-scene-model" || call.Prompt.SessionID != request.SessionID {
|
||||
t.Fatal("producer/validator native selections lost")
|
||||
}
|
||||
}
|
||||
initial, revised := client.requests[0].Prompt.Messages, client.requests[2].Prompt.Messages
|
||||
if len(revised) != len(initial)+2 || !reflect.DeepEqual(initial, revised[:len(initial)]) || revised[len(initial)].Role != "assistant" || revised[len(initial)].Content != narrativeScene {
|
||||
t.Fatal("continuation includes wrong history or regenerated context")
|
||||
}
|
||||
if !strings.Contains(revised[len(initial)+1].Content, "Return kind: combat for this scene.") || !strings.Contains(revised[len(initial)+1].Content, "Bandits attack the party") {
|
||||
t.Fatal("combat judgment guidance lost")
|
||||
}
|
||||
if _, err := scenecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermissiveSceneRetainsOnlyFinalUsableCandidateWithoutSemanticRevalidation(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
// Both scenes remain non-combat, but the final title/summary must win.
|
||||
final := `{"kind":"narrative","title":"Final watchtower description","summary":"The party is attacked by bandits."}`
|
||||
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment, final, combatJudgment}}
|
||||
extractor := newExtractor(t, client)
|
||||
calls := bindSceneAssessment(t, extractor, &request, true)
|
||||
result, err := extractor.Extract(context.Background(), request)
|
||||
if err != nil || result.Outcome.Disposition != contracts.ProducerFallback || result.Outcome.ReuseEligible || result.Outcome.FallbackReason == "" || len(client.requests) != 4 || len(*calls) != 8 || len(result.Value.Scenes) != 1 {
|
||||
t.Fatalf("semantic fallback=%#v %v, native=%d assessments=%#v", result, err, len(client.requests), *calls)
|
||||
}
|
||||
scene := result.Value.Scenes[0]
|
||||
if scene.Title != "Final watchtower description" || scene.Kind != dnd.SceneKindNarrative || scene.SourceRef != request.Chunk.Ref || scene.ID != request.Chunk.ID {
|
||||
t.Fatalf("fallback recovered earlier scene or invented classification=%#v", scene)
|
||||
}
|
||||
if _, err := scenecodec.New().Encode(result.Value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSceneCannotFallbackAfterMissingOrIntrinsicallyUnusableFinalResponse(t *testing.T) {
|
||||
failure := errors.New("provider unavailable")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
responses []string
|
||||
failures []error
|
||||
wantAssessments int
|
||||
}{
|
||||
{"no first response", nil, []error{failure}, 0},
|
||||
{"structural failure", []string{`{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 0},
|
||||
{"blank final scene", []string{blankScene, blankScene}, nil, 2},
|
||||
{"failed correction", []string{blankScene}, []error{nil, failure}, 1},
|
||||
// An initially usable scene cannot be recovered after a result-less correction.
|
||||
{"lost usable prior scene", []string{narrativeScene}, []error{nil, failure}, 1},
|
||||
{"structurally unusable revision", []string{narrativeScene, `{"kind":"unknown","title":"Bad","summary":"Bad"}`}, nil, 1},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
request.Operation.Controls.MaxAdditionalTurns = 1
|
||||
assessments := 0
|
||||
request.Operation.Assess = func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||
assessments++
|
||||
return contracts.CandidateAssessment{Validators: []contracts.ValidatorAssessment{{ValidatorName: "test/reject", Decision: contracts.AssessmentRejected,
|
||||
Result: contracts.ValidationResult{ReasonCode: "test_rejection", Message: "Scene rejected.", CorrectionGuidance: "Correct the scene description."}}}}, nil
|
||||
}
|
||||
client := &fakeSceneDescriptionsLLMClient{responses: test.responses, errors: test.failures}
|
||||
if len(test.responses) == 1 {
|
||||
client.raw = test.responses[0]
|
||||
}
|
||||
result, err := newExtractor(t, client).Extract(context.Background(), request)
|
||||
if err == nil || result.Outcome.Disposition != "" || len(result.Value.Scenes) != 0 || assessments != test.wantAssessments {
|
||||
t.Fatalf("unusable scene=%#v %v, assessments=%d", result, err, assessments)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStrictSceneRejectionAndCancellationDoNotPublishFallback(t *testing.T) {
|
||||
request := extractionRequest()
|
||||
request.Operation.Controls.MaxAdditionalTurns = 0
|
||||
client := &fakeSceneDescriptionsLLMClient{responses: []string{narrativeScene, combatJudgment}}
|
||||
extractor := newExtractor(t, client)
|
||||
bindSceneAssessment(t, extractor, &request, true)
|
||||
result, err := extractor.Extract(context.Background(), request)
|
||||
if err == nil || result.Outcome.Disposition != "" || len(client.requests) != 2 {
|
||||
t.Fatalf("strict scene=%#v %v", result, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
request.Operation.Controls.Mode = contracts.ModePermissive
|
||||
canceled := &fakeSceneDescriptionsLLMClient{raw: combatScene, onGenerate: func(context.Context, promptkit.GenerateRequest) { cancel() }}
|
||||
result, err = newExtractor(t, canceled).Extract(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result.Outcome.Disposition != "" || len(canceled.requests) != 1 {
|
||||
t.Fatalf("cancellation=%#v %v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
|
||||
rootassets "gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
)
|
||||
|
||||
const promptAssetRoot = "assets/prompts"
|
||||
|
||||
var promptAssetManifest = shared.PromptAssetManifest{
|
||||
ModuleDir: "dnd.scene_descriptions",
|
||||
ModuleFiles: []promptfs.ModulePromptFile{
|
||||
{Name: "prompt.yaml", Path: "prompts/prompt.yaml"},
|
||||
{Name: "instructions.md", Path: "prompts/instructions.md"},
|
||||
},
|
||||
SharedFiles: []string{
|
||||
"common-dnd-system.md",
|
||||
"common-dnd-identity.md",
|
||||
"common-dnd-references.md",
|
||||
"common-dnd-transcript-chunk.md",
|
||||
"common-dnd-scene-combat-policy.md",
|
||||
},
|
||||
}
|
||||
|
||||
func moduleAssetFS() (fs.FS, error) {
|
||||
assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scope scene-description assets: %w", err)
|
||||
}
|
||||
return assets, nil
|
||||
}
|
||||
|
||||
func RegisterPromptAssets(registry *llm.AssetRegistry) error {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
promptFS, err := promptAssetManifest.PromptFS(assets)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare scene-description prompt assets: %w", err)
|
||||
}
|
||||
if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
schemas, err := fs.Sub(assets, "schemas")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scope scene-description schemas: %w", err)
|
||||
}
|
||||
return registry.RegisterSchemaFS(schemas, ".")
|
||||
}
|
||||
|
||||
func promptAssetMetadata() (string, error) {
|
||||
promptAssetHashOnce.Do(func() {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
promptAssetHashErr = err
|
||||
return
|
||||
}
|
||||
promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets)
|
||||
})
|
||||
return promptAssetHash, promptAssetHashErr
|
||||
}
|
||||
|
||||
var (
|
||||
promptAssetHashOnce sync.Once
|
||||
promptAssetHash string
|
||||
promptAssetHashErr error
|
||||
)
|
||||
@@ -1,71 +0,0 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/testsupport/promptkitfixture"
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestRegisterPromptAssetsPreparesSceneDescriptionPrompt(t *testing.T) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := RegisterPromptAssets(registry); err != nil {
|
||||
t.Fatalf("RegisterPromptAssets() error = %v, want nil", err)
|
||||
}
|
||||
engine := promptkitfixture.NewEngine(t, registry, promptkit.Config{Timeout: time.Second}, map[string]promptkit.ProfileConfig{
|
||||
"scene-description-test-profile": promptkitfixture.Profile("http://127.0.0.1:1/v1", "scene-description-test-model"),
|
||||
})
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "scene-description-test-profile",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"scene-description-transcript"}`),
|
||||
"players": promptkit.Inline("scene-description-player"),
|
||||
"party": promptkit.Inline(" "),
|
||||
"glossary": promptkit.Inline(" "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_descriptions_llm.v1.json" {
|
||||
t.Fatalf("prepared prompt = %#v, want scene-description prompt identity and schema wiring", prepared)
|
||||
}
|
||||
const combatPolicySentinel = "substantive active combat materially organizes the"
|
||||
transcriptIndex := -1
|
||||
policyIndex := -1
|
||||
instructionsIndex := len(prepared.Messages) - 1
|
||||
policyOccurrences := 0
|
||||
for index, message := range prepared.Messages {
|
||||
if strings.Contains(message.Content, "scene-description-transcript") {
|
||||
transcriptIndex = index
|
||||
}
|
||||
if count := strings.Count(message.Content, combatPolicySentinel); count > 0 {
|
||||
policyIndex = index
|
||||
policyOccurrences += count
|
||||
}
|
||||
}
|
||||
if policyOccurrences != 1 || policyIndex <= transcriptIndex || policyIndex >= instructionsIndex {
|
||||
t.Fatalf("combat policy appeared %d times at message %d; want once after transcript message %d and before instructions message %d", policyOccurrences, policyIndex, transcriptIndex, instructionsIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromptMetadataAndDiagnosticsDoNotContainRawAssets(t *testing.T) {
|
||||
hash, err := promptAssetMetadata()
|
||||
if err != nil || !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err)
|
||||
}
|
||||
payload, err := json.Marshal(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, forbidden := range []string{"common-dnd-system", "dnd_scene_descriptions_llm.v1.json"} {
|
||||
if strings.Contains(string(payload), forbidden) {
|
||||
t.Fatalf("metadata leaked raw prompt/schema content %q: %s", forbidden, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,10 +11,10 @@ import (
|
||||
)
|
||||
|
||||
func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "LLM client") {
|
||||
if _, err := New(nil, Options{}); err == nil || !strings.Contains(err.Error(), "native engine") {
|
||||
t.Fatalf("New(nil) error = %v, want dependency error", err)
|
||||
}
|
||||
if _, err := New(&fakeSceneDescriptionsLLMClient{}, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
if _, err := New(newExtractor(t, &fakeSceneDescriptionsLLMClient{}).engine, Options{}, contracts.ReferenceSet{}, contracts.ReferenceSet{}); err == nil || !strings.Contains(err.Error(), "at most one reference set") {
|
||||
t.Fatalf("New() error = %v, want reference-set error", err)
|
||||
}
|
||||
if got := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).Key(); got != Key {
|
||||
@@ -24,7 +24,7 @@ func TestNewRequiresLLMClientAndRejectsAmbiguousReferences(t *testing.T) {
|
||||
|
||||
func TestModuleSpecAndReferenceSlots(t *testing.T) {
|
||||
want := pipeline.ModuleSpec{
|
||||
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
Key: Key, Stage: pipeline.StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{PromptID, CorrectionPromptID}, Requires: []string{"chunks", "source.transcript"}, Provides: []string{"dnd.scene_descriptions"}, ArtifactKind: dnd.SceneDescriptionListKind,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "glossary", Description: "Optional campaign glossary reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
{Name: "party", Description: "Optional party roster reference material used only to disambiguate scene descriptions.", AcceptedMediaTypes: []string{"application/json", "application/x-yaml", "application/yaml", "text/markdown", "text/plain"}},
|
||||
@@ -61,32 +61,3 @@ func TestRegisterStoresTypedModuleSpecAndOptionsAreStrict(t *testing.T) {
|
||||
t.Fatalf("DecodeOptions() error = %v, want strict options error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorMetadataAndCheckpointIdentity(t *testing.T) {
|
||||
metadata := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).ManifestMetadata()
|
||||
for key, want := range map[string]string{
|
||||
"prompt_id": PromptID, "prompt_version": SchemaVersion,
|
||||
"response_schema_key": string(ResponseSchemaKey), "response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName, "response_schema_version": SchemaVersion,
|
||||
"mapping_policy": mappingPolicy,
|
||||
} {
|
||||
if metadata[key] != want {
|
||||
t.Fatalf("metadata[%q] = %#v, want %q", key, metadata[key], want)
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"prompt_sha256", "response_schema_sha256"} {
|
||||
if value, ok := metadata[key].(string); !ok || !strings.HasPrefix(value, "sha256:") {
|
||||
t.Fatalf("metadata[%q] = %#v, want hash", key, metadata[key])
|
||||
}
|
||||
}
|
||||
want := map[string]string{"prompt": metadata["prompt_sha256"].(string), "response_schema": metadata["response_schema_sha256"].(string), "mapping_policy": mappingPolicy}
|
||||
fingerprints := newExtractor(t, &fakeSceneDescriptionsLLMClient{}).CheckpointFingerprints()
|
||||
if len(fingerprints) != len(want) {
|
||||
t.Fatalf("CheckpointFingerprints() = %#v, want %d entries", fingerprints, len(want))
|
||||
}
|
||||
for _, fingerprint := range fingerprints {
|
||||
if fingerprint.Value != want[fingerprint.Name] {
|
||||
t.Fatalf("fingerprint %q = %q, want %#v", fingerprint.Name, fingerprint.Value, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package scenedescriptions
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
|
||||
const (
|
||||
PromptID = "dnd.scene_descriptions"
|
||||
ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_descriptions_llm")
|
||||
ResponseSchemaID = "notarius.dnd.scene_descriptions.llm"
|
||||
ResponseSchemaName = "notarius_dnd_scene_descriptions_llm_v1"
|
||||
SchemaVersion = "v1"
|
||||
mappingPolicy = "dnd.scene_descriptions.mapping.v1"
|
||||
)
|
||||
|
||||
func loadResponseSchema() (llm.ResponseSchema, error) {
|
||||
assets, err := moduleAssetFS()
|
||||
if err != nil {
|
||||
return llm.ResponseSchema{}, err
|
||||
}
|
||||
return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{
|
||||
Key: ResponseSchemaKey,
|
||||
ID: ResponseSchemaID,
|
||||
Version: SchemaVersion,
|
||||
Name: ResponseSchemaName,
|
||||
AssetPath: "schemas/dnd_scene_descriptions_llm.v1.json",
|
||||
})
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package scenedescriptions
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatalf("loadResponseSchema() error = %v, want nil", err)
|
||||
}
|
||||
if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) {
|
||||
t.Fatalf("schema = %#v, want private scene-description schema identity", schema)
|
||||
}
|
||||
valid := map[string]any{"kind": "narrative", "title": "Arrival", "summary": "The party enters the tower."}
|
||||
if err := validateJSONSchema(t, valid, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("valid private response rejected: %v", err)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
response map[string]any
|
||||
valid bool
|
||||
}{
|
||||
{name: "missing required field", response: map[string]any{"kind": "combat", "title": "Ambush"}},
|
||||
{name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}},
|
||||
{name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}},
|
||||
{name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}},
|
||||
{name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true},
|
||||
{name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true},
|
||||
{name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}},
|
||||
{name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}},
|
||||
{name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := validateJSONSchema(t, test.response, schema.JSONSchema)
|
||||
if (err == nil) != test.valid {
|
||||
t.Fatalf("validateJSONSchema() error = %v, want valid=%t", err, test.valid)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := validateJSONSchemaContent([]byte(`{"kind":`), schema.JSONSchema); err == nil {
|
||||
t.Fatal("validateJSONSchemaContent() error = nil, want malformed JSON rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) {
|
||||
first, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.JSONSchema[0] = '['
|
||||
second, err := loadResponseSchema()
|
||||
if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) {
|
||||
t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err)
|
||||
}
|
||||
if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil {
|
||||
t.Fatalf("schema diagnostics included raw content: %#v", diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaAcceptsEverySupportedKind(t *testing.T) {
|
||||
schema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, kind := range []string{string(dnd.SceneKindCombat), string(dnd.SceneKindNarrative), string(dnd.SceneKindRecap), string(dnd.SceneKindMeta)} {
|
||||
response := map[string]any{"kind": kind, "title": "Title", "summary": "Summary"}
|
||||
if err := validateJSONSchema(t, response, schema.JSONSchema); err != nil {
|
||||
t.Fatalf("supported scene kind %q was rejected: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func validateJSONSchema(t *testing.T, instance map[string]any, schemaContent []byte) error {
|
||||
t.Helper()
|
||||
content, err := json.Marshal(instance)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return validateJSONSchemaContent(content, schemaContent)
|
||||
}
|
||||
|
||||
func validateJSONSchemaContent(instanceContent, schemaContent []byte) error {
|
||||
instance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
schemaDocument, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := compiler.AddResource("schema.json", schemaDocument); err != nil {
|
||||
return err
|
||||
}
|
||||
compiled, err := compiler.Compile("schema.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return compiled.Validate(instance)
|
||||
}
|
||||
@@ -3,14 +3,19 @@ package scenedescriptions
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/assets"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
)
|
||||
|
||||
func extractionRequest() contracts.TypedExtractionRequest {
|
||||
func extractionRequest() contracts.TypedExtractionRequest[dnd.SceneDescriptionList] {
|
||||
doc := sourceDocument()
|
||||
chunk := &source.Chunk{
|
||||
ID: "session-alpha:chunk:0",
|
||||
@@ -22,12 +27,15 @@ func extractionRequest() contracts.TypedExtractionRequest {
|
||||
Units: append([]source.SourceUnit(nil), doc.Units...),
|
||||
Metadata: map[string]any{"ignored": "chunk metadata"},
|
||||
}
|
||||
return contracts.TypedExtractionRequest{
|
||||
return contracts.TypedExtractionRequest[dnd.SceneDescriptionList]{
|
||||
Source: doc,
|
||||
Chunk: chunk,
|
||||
SourceInput: contracts.NewLLMInputMaterial("source", chunk.MediaType, chunk.Content, "sha256:chunk", "file:///session-alpha.json"),
|
||||
SessionID: "session-123",
|
||||
LLMProfile: "profile-scene-descriptions",
|
||||
Operation: contracts.ProducerOperation[dnd.SceneDescriptionList]{Controls: contracts.ProducerControls{Mode: contracts.ModeStrict, Cache: contracts.CacheBypass}, ReuseEligible: true, Assess: func(context.Context, dnd.SceneDescriptionList) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,58 +53,99 @@ func sourceDocument() *source.SourceDocument {
|
||||
}
|
||||
}
|
||||
|
||||
func newExtractor(t *testing.T, client contracts.StructuredLLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
extractor, err := New(client, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatalf("New() error = %v, want nil", err)
|
||||
}
|
||||
return extractor
|
||||
}
|
||||
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest) contracts.TypedExtractionRequest {
|
||||
func mismatchedSourceInputRequest(req contracts.TypedExtractionRequest[dnd.SceneDescriptionList]) contracts.TypedExtractionRequest[dnd.SceneDescriptionList] {
|
||||
req.SourceInput = contracts.NewLLMInputMaterial("source", "application/json", []byte(`{"different":true}`), "sha256:other", "file:///other.json")
|
||||
return req
|
||||
}
|
||||
|
||||
func cloneStructuredCompletionRequest(req contracts.StructuredCompletionRequest) contracts.StructuredCompletionRequest {
|
||||
req.Inputs = req.Inputs.Clone()
|
||||
correction, err := contracts.CloneSemanticCorrection(req.Correction)
|
||||
func newExtractor(t *testing.T, client promptkit.LLMClient, references ...contracts.ReferenceSet) *Extractor {
|
||||
t.Helper()
|
||||
type settings struct {
|
||||
PromptKit promptkit.Settings `yaml:"promptkit"`
|
||||
}
|
||||
definition := promptkit.Configure(configkit.Definition[settings]{}, func(value *settings) *promptkit.Settings { return &value.PromptKit })
|
||||
filesystem := configkit.NewFilesystem("Notarius assets", assets.FS())
|
||||
result, err := configkit.LoadLayered(configkit.Input{Source: configkit.Bytes("occurrence fixture", []byte(`promptkit:
|
||||
profiles:
|
||||
- id: profile-scene-descriptions
|
||||
endpoint: https://example.invalid/v1
|
||||
model: occurrence-test-model
|
||||
api_key_required: false
|
||||
- id: scene-validator-profile
|
||||
endpoint: https://example.invalid/v1
|
||||
model: validator-test-model
|
||||
api_key_required: false
|
||||
`))}, configkit.LayeredOptions[settings]{Definition: definition, Underlays: []configkit.Underlay{
|
||||
promptkit.BundledOpenRouterUnderlay(),
|
||||
{Inputs: []configkit.Input{{Source: filesystem.File("notarius.yml"), Imports: true, Filesystem: &configkit.FilesystemContext{FS: &filesystem, Root: "."}}}},
|
||||
}})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Correction = correction
|
||||
if len(req.Vars) == 0 {
|
||||
req.Vars = nil
|
||||
return req
|
||||
engine, err := promptkit.NewFromConfig(result, promptkit.RuntimeConfig{}, promptkit.WithLLMClient(client))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vars := make(map[string]any, len(req.Vars))
|
||||
for key, value := range req.Vars {
|
||||
vars[key] = value
|
||||
extractor, err := New(engine, Options{}, references...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Vars = vars
|
||||
return req
|
||||
return extractor
|
||||
}
|
||||
|
||||
func renderedInput(t *testing.T, request promptkit.GenerateRequest, name string) []byte {
|
||||
t.Helper()
|
||||
key := map[string]string{"transcript": "units", "npc_registry": "npcs", "location_registry": "locations"}[name]
|
||||
for _, message := range request.Prompt.Messages {
|
||||
start := strings.Index(message.Content, "{")
|
||||
if start < 0 {
|
||||
continue
|
||||
}
|
||||
content := []byte(message.Content[start:])
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(content, &fields) == nil && fields[key] != nil {
|
||||
return content
|
||||
}
|
||||
}
|
||||
t.Fatalf("missing rendered %s input", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeSceneDescriptionsLLMClient struct {
|
||||
response extractionResponse
|
||||
err error
|
||||
requests []contracts.StructuredCompletionRequest
|
||||
response extractionResponse
|
||||
responses []string
|
||||
raw string
|
||||
err error
|
||||
errors []error
|
||||
requests []promptkit.GenerateRequest
|
||||
onGenerate func(context.Context, promptkit.GenerateRequest)
|
||||
}
|
||||
|
||||
func (client *fakeSceneDescriptionsLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.requests = append(client.requests, cloneStructuredCompletionRequest(req))
|
||||
func (client *fakeSceneDescriptionsLLMClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
index := len(client.requests)
|
||||
client.requests = append(client.requests, request)
|
||||
if client.onGenerate != nil {
|
||||
client.onGenerate(ctx, request)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
return nil, client.err
|
||||
}
|
||||
target, ok := out.(*extractionResponse)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output target")
|
||||
if index < len(client.errors) && client.errors[index] != nil {
|
||||
return nil, client.errors[index]
|
||||
}
|
||||
*target = client.response
|
||||
content, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
content := client.raw
|
||||
if index < len(client.responses) {
|
||||
content = client.responses[index]
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content}, nil
|
||||
if content == "" {
|
||||
encoded, err := json.Marshal(client.response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = string(encoded)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: content}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user