Files
audita/internal/prompts/registry_test.go

121 lines
5.1 KiB
Go

package prompts
import (
"context"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
promptInputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
if err != nil {
t.Fatal(err)
}
endpoint := "https://example.invalid/v1"
model := "fixture-model"
sources := pk.ConfigurationSources{
Underlays: []ck.Underlay{
{Inputs: promptInputs},
{Inputs: []ck.Input{{Source: pk.ProfileSource("registry fixture profile", []pk.ProfileConfig{
{ID: "fixture", Endpoint: &endpoint, Model: &model},
})}}},
},
}
engine, err := pk.NewEngine(pk.Config{Sources: sources},
pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot),
)
if err != nil {
t.Fatal(err)
}
vars := map[string]string{
"HardeningPolicy": HardeningText(),
"TranscriptDescriptionBlock": "Transcript description:\nfixture-description\n\n",
"GlossaryJSON": "{\"name\":\"fixture-glossary\"}",
"SectionJSON": "{\"text\":\"fixture-section\"}",
"PayloadJSON": "[{\"text\":\"fixture-correction\"}]",
}
vars = WithSharedVariables(vars)
for _, tc := range []struct{ id, schema, payload string }{
{PromptIDModuleGlossaryProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleHomophonesProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleSpokenWordProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDModuleGrammarProposal, "correction_set.v1.json", "fixture-section"},
{PromptIDValidatorSpokenFormPlausibility, "validator_decision_set.v1.json", "fixture-correction"},
{PromptIDValidatorMeaningReversalReview, "validator_decision_set.v1.json", "fixture-correction"},
{PromptIDValidatorEditorialReview, "validator_decision_set.v1.json", "fixture-correction"},
} {
t.Run(tc.id, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: vars})
if err != nil {
t.Fatal(err)
}
if prepared.PromptID != tc.id || prepared.OutputContract.SchemaPath != tc.schema || prepared.StructuredOutput == nil {
t.Fatalf("selection/schema: %+v", prepared)
}
if len(prepared.Messages) != 2 {
t.Fatalf("message count: %d", len(prepared.Messages))
}
rendered := prepared.Messages[0].Content + prepared.Messages[1].Content
for _, fragment := range []string{"fixture-description", tc.payload} {
if !strings.Contains(rendered, fragment) {
t.Errorf("missing %q", fragment)
}
}
if strings.HasPrefix(tc.id, "audita/modules.") && !strings.Contains(rendered, "fixture-glossary") {
t.Error("missing glossary context")
}
missing := map[string]string{"HardeningPolicy": HardeningText()}
if _, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: tc.id, ProfileID: "fixture", Vars: missing}); err == nil {
t.Error("missing variables accepted")
}
})
}
}
func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) {
all := RegisteredMetadata()
if len(all) != 11 {
t.Fatalf("prompt count: %d", len(all))
}
for _, meta := range all {
if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" {
t.Fatalf("metadata: %+v", meta)
}
if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil {
t.Fatal(err)
}
}
}
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {
inputs, err := pk.FilesystemInputs("embedded prompts", Filesystem(), AssetRoot)
if err != nil {
t.Fatal(err)
}
endpoint, model := "https://example.invalid/v1", "fixture"
engine, err := pk.NewEngine(pk.Config{Sources: pk.ConfigurationSources{Underlays: []ck.Underlay{{Inputs: inputs}, {Inputs: []ck.Input{{Source: pk.ProfileSource("fixture", []pk.ProfileConfig{{ID: "fixture", Endpoint: &endpoint, Model: &model}})}}}}}}, pk.WithSchemaFS(responseschema.Filesystem(), responseschema.AssetRoot))
if err != nil {
t.Fatal(err)
}
for _, id := range []string{PromptIDModuleGlossaryRepair, PromptIDModuleHomophonesRepair, PromptIDModuleSpokenWordRepair, PromptIDModuleGrammarRepair} {
t.Run(id, func(t *testing.T) {
prepared, err := engine.Prepare(context.Background(), pk.RunRequest{PromptID: id, ProfileID: "fixture", Vars: WithSharedVariables(map[string]string{"RepairRequestJSON": `{"proposal_ref":2,"correction":{"id":1,"original_text":" ","corrected_text":""},"findings":[{"validator":"shape","reason":"example"}]}`})})
if err != nil {
t.Fatal(err)
}
if prepared.OutputContract.SchemaPath != "correction_repair.v1.json" || prepared.StructuredOutput == nil || len(prepared.Messages) != 1 || prepared.Messages[0].Role != pk.RoleUser {
t.Fatalf("prepared = %+v", prepared)
}
for _, fragment := range []string{`{"proposal_ref":2,"correction":{"id":1,"original_text":" ","corrected_text":""},"findings":[{"validator":"shape","reason":"example"}]}`} {
if !strings.Contains(prepared.Messages[0].Content, fragment) {
t.Errorf("missing %q", fragment)
}
}
})
}
}