Files
audita/internal/prompts/registry_test.go

129 lines
5.7 KiB
Go

package prompts
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
promptInputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
if err != nil {
t.Fatal(err)
}
endpoint := "https://example.invalid/v1"
model := "fixture-model"
sources := pk.ConfigurationSources{
Underlays: []ck.Underlay{
{Inputs: promptInputs},
{Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", []pk.ProfileConfig{
{ID: "fixture", Endpoint: &endpoint, Model: &model},
})}}},
},
}
engine, err := pk.NewEngine(pk.Config{Sources: sources},
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
)
if err != nil {
t.Fatal(err)
}
vars := map[string]string{
"HardeningPolicy": HardeningText(),
"TranscriptDescriptionBlock": "Transcript description:\nfixture-description\n\n",
"GlossaryJSON": "{\"name\":\"fixture-glossary\"}",
"SectionJSON": "{\"text\":\"fixture-section\"}",
"PayloadJSON": "[{\"text\":\"fixture-correction\"}]",
}
vars = WithSharedVariables(vars)
for _, tc := range []struct{ id, schema, payload string }{
{PromptIDModuleGlossaryProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleHomophonesProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleSpokenWordProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleGrammarProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDValidatorSpokenFormPlausibility, "validator_decision_set.v1.json", "fixture-correction"},
{PromptIDValidatorMeaningReversalReview, "validator_decision_set.v1.json", "fixture-correction"},
{PromptIDValidatorEditorialReview, "validator_decision_set.v1.json", "fixture-correction"},
} {
t.Run(tc.id, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: vars})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != tc.id || prepared.OutputContract.SchemaPath != tc.schema || prepared.StructuredOutput == nil {
t.Fatalf("selection/schema: %+v", prepared)
}
if len(prepared.Messages) != 2 {
t.Fatalf("message count: %d", len(prepared.Messages))
}
rendered := prepared.Messages[0].Content + prepared.Messages[1].Content
for _, fragment := range []string{"fixture-description", tc.payload} {
if !strings.Contains(rendered, fragment) {
t.Errorf("missing %q", fragment)
}
}
if strings.HasPrefix(tc.id, "audita/modules.") && !strings.Contains(rendered, "fixture-glossary") {
t.Error("missing glossary context")
}
missing := map[string]string{"HardeningPolicy": HardeningText()}
if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: missing}); err == nil {
t.Error("missing variables accepted")
}
})
}
}
func TestRegistryMetadataIdentifiesRequiredPrompts(t *testing.T) {
required := []string{
"audita/modules.glossary.proposal", "audita/modules.glossary.repair",
"audita/modules.grammar.proposal", "audita/modules.grammar.repair",
"audita/modules.homophones.proposal", "audita/modules.homophones.repair",
"audita/modules.spoken_word.proposal", "audita/modules.spoken_word.repair",
"audita/validators.editorial_review", "audita/validators.meaning_reversal_review",
"audita/validators.spoken_form_plausibility",
}
for _, id := range required {
metadata, ok := LookupMetadata(id)
if !ok || metadata.PromptID != id || metadata.PromptSource != SourceBuiltin ||
metadata.EmbeddedPath != AssetRoot+"/"+strings.TrimPrefix(id, "audita/")+".yaml" {
t.Fatalf("required prompt metadata for %q: %+v", id, metadata)
}
}
all := RegisteredMetadata()
all[0].PromptID = "changed"
if again := RegisteredMetadata(); again[0].PromptID == "changed" {
t.Fatal("metadata enumeration shares mutable state")
}
}
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {
inputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
if err != nil {
t.Fatal(err)
}
endpoint, model := "https://example.invalid/v1", "fixture"
engine, err := pk.NewEngine(pk.Config{Sources: pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: inputs}, {Inputs: []ck.Input{{Source: pk.ProfileSource("fixture", []pk.ProfileConfig{{ID: "fixture", Endpoint: &endpoint, Model: &model}})}}}}}}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err != nil {
t.Fatal(err)
}
for _, id := range []string{PromptIDModuleGlossaryRepair, PromptIDModuleHomophonesRepair, PromptIDModuleSpokenWordRepair, PromptIDModuleGrammarRepair} {
t.Run(id, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: id, ProfileID: "fixture", Vars: WithSharedVariables(map[string]string{"RepairRequestJSON": `{"proposal_ref":2,"correction":{"id":1,"original_text":" ","corrected_text":""},"findings":[{"validator":"shape","reason":"example"}]}`})})
if err != nil {
t.Fatal(err)
}
if prepared.OutputContract.SchemaPath != "correction_repair.v1.json" || prepared.StructuredOutput == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Role != pk.RoleUser {
t.Fatalf("prepared = %+v", prepared)
}
for _, fragment := range []string{`{"proposal_ref":2,"correction":{"id":1,"original_text":" ","corrected_text":""},"findings":[{"validator":"shape","reason":"example"}]}`} {
if !strings.Contains(prepared.Messages[0].Content, fragment) {
t.Errorf("missing %q", fragment)
}
}
})
}
}