Capture complete prompt resources for inspection
This commit is contained in:
@@ -309,6 +309,10 @@ backend/profile references into nested reports or require execution readiness.
|
|||||||
|
|
||||||
## Stage 4: Capture Complete Prompt Resource Information
|
## Stage 4: Capture Complete Prompt Resource Information
|
||||||
|
|
||||||
|
**Completed:** Captured prompt resources retain inherited scalar presence and
|
||||||
|
materialized message text independently of requirement metadata. Focused
|
||||||
|
configuration tests pass, including inherited file content and explicit values.
|
||||||
|
|
||||||
**Scope:** Prompt presence and resource projection, without changing execution
|
**Scope:** Prompt presence and resource projection, without changing execution
|
||||||
or requirement-discovery representations.
|
or requirement-discovery representations.
|
||||||
|
|
||||||
|
|||||||
@@ -120,6 +120,7 @@ const (
|
|||||||
type loaded struct {
|
type loaded struct {
|
||||||
result *configkit.Result[document]
|
result *configkit.Result[document]
|
||||||
promptMetadata map[string][]promptInputPresence
|
promptMetadata map[string][]promptInputPresence
|
||||||
|
promptRepair map[string]bool
|
||||||
prompts map[string]*promptEntry
|
prompts map[string]*promptEntry
|
||||||
registry *backend.Registry
|
registry *backend.Registry
|
||||||
backendRecords map[string]backendConfig
|
backendRecords map[string]backendConfig
|
||||||
@@ -234,7 +235,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
Sensitive: secretProviderPath,
|
Sensitive: secretProviderPath,
|
||||||
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
Resolve: func(config *document, resolution *configkit.Resolution) error {
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata)
|
capturePromptMetadata(resources.prompts, resources.bases.prompts, resources.namespace, resolution, state.promptMetadata, state.promptRepair)
|
||||||
}
|
}
|
||||||
for _, resources := range resourceSets(config) {
|
for _, resources := range resourceSets(config) {
|
||||||
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
resolveBackends(resources.backends, resources.bases.backends, resolution)
|
||||||
@@ -253,6 +254,7 @@ func configurationDefinition(state *loaded) configkit.Definition[document] {
|
|||||||
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
|
||||||
state := &loaded{
|
state := &loaded{
|
||||||
promptMetadata: make(map[string][]promptInputPresence),
|
promptMetadata: make(map[string][]promptInputPresence),
|
||||||
|
promptRepair: make(map[string]bool),
|
||||||
}
|
}
|
||||||
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
|
||||||
Underlays: underlays,
|
Underlays: underlays,
|
||||||
@@ -317,11 +319,12 @@ func checkConfigurationVersion(version configkit.Version, validation *configkit.
|
|||||||
}
|
}
|
||||||
|
|
||||||
type promptInputPresence struct {
|
type promptInputPresence struct {
|
||||||
|
required fieldState
|
||||||
contentType fieldState
|
contentType fieldState
|
||||||
description fieldState
|
description fieldState
|
||||||
}
|
}
|
||||||
|
|
||||||
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence) {
|
func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, namespace string, resolution *configkit.Resolution, destination map[string][]promptInputPresence, repair map[string]bool) {
|
||||||
for promptIndex, prompt := range prompts {
|
for promptIndex, prompt := range prompts {
|
||||||
inputs := prompt.Inputs
|
inputs := prompt.Inputs
|
||||||
states := make([]promptInputPresence, len(inputs))
|
states := make([]promptInputPresence, len(inputs))
|
||||||
@@ -329,11 +332,13 @@ func capturePromptMetadata(prompts []promptConfig, basePath configkit.Path, name
|
|||||||
for index := range inputs {
|
for index := range inputs {
|
||||||
path := base.Index(index)
|
path := base.Index(index)
|
||||||
states[index] = promptInputPresence{
|
states[index] = promptInputPresence{
|
||||||
|
required: captureFieldState(path.Key("required"), resolution),
|
||||||
contentType: captureFieldState(path.Key("content_type"), resolution),
|
contentType: captureFieldState(path.Key("content_type"), resolution),
|
||||||
description: captureFieldState(path.Key("description"), resolution),
|
description: captureFieldState(path.Key("description"), resolution),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
destination[indexedResourceID(namespace, prompt.ID)] = states
|
destination[indexedResourceID(namespace, prompt.ID)] = states
|
||||||
|
repair[indexedResourceID(namespace, prompt.ID)] = captureFieldState(basePath.Index(promptIndex).Key("output").Key("repair_attempts"), resolution) == fieldValue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type promptEntry struct {
|
type promptEntry struct {
|
||||||
|
resource promptConfig
|
||||||
definition domain.PromptDefinition
|
definition domain.PromptDefinition
|
||||||
inspection domain.PromptInspection
|
inspection domain.PromptInspection
|
||||||
requirementInputs []prompt.InputMetadata
|
requirementInputs []prompt.InputMetadata
|
||||||
@@ -86,7 +87,7 @@ func (state *loaded) materializePrompt(id string, raw promptConfig, base configk
|
|||||||
}
|
}
|
||||||
definition.Inputs = inputs
|
definition.Inputs = inputs
|
||||||
definition.Templates = messages
|
definition.Templates = messages
|
||||||
entry := &promptEntry{requirementInputs: inputMetadata, messages: compiledMessages}
|
entry := &promptEntry{resource: raw, requirementInputs: inputMetadata, messages: compiledMessages}
|
||||||
definition.Validation = contract
|
definition.Validation = contract
|
||||||
if raw.SessionID != nil {
|
if raw.SessionID != nil {
|
||||||
var err error
|
var err error
|
||||||
@@ -316,8 +317,8 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id string) (*domain.
|
|||||||
return ownPromptDefinition(entry.definition), entry.hash, nil
|
return ownPromptDefinition(entry.definition), entry.hash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPromptInspection returns an independently owned inspection of one
|
// GetPromptInspection returns lightweight metadata used for requirement
|
||||||
// captured prompt without copying its execution templates.
|
// selection, independently of the full resource inspection view.
|
||||||
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
func (state *loaded) GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -84,6 +84,10 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
files["catalog/body.md"].Data = []byte("Changed")
|
files["catalog/body.md"].Data = []byte("Changed")
|
||||||
files["app/body.md"].Data = []byte("Changed replacement")
|
files["app/body.md"].Data = []byte("Changed replacement")
|
||||||
files["other/body.md"].Data = []byte("Changed other collection")
|
files["other/body.md"].Data = []byte("Changed other collection")
|
||||||
|
resource, err := state.InspectPrompt(context.Background(), "docs/child")
|
||||||
|
if err != nil || resource.Messages[0].Content != "Captured {{.name}}" || resource.Messages[0].CacheControl == nil || resource.Messages[0].CacheControl.TTL == nil || *resource.Messages[0].CacheControl.TTL != "1h" || resource.Inputs == nil || (*resource.Inputs)[0].Required != nil {
|
||||||
|
t.Fatalf("captured resource = %+v, %v", resource, err)
|
||||||
|
}
|
||||||
again, againInspection, err := state.prompt("docs/child")
|
again, againInspection, err := state.prompt("docs/child")
|
||||||
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
if err != nil || again.Templates[0].Content != "Captured {{.name}}" || againInspection.PromptHash != inspection.PromptHash {
|
||||||
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
t.Fatalf("captured definition changed after source mutation: %+v, %+v, %v", again, againInspection, err)
|
||||||
@@ -99,6 +103,25 @@ func TestLoadCapturesInheritedImportedContent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestInspectPromptResourcePresence(t *testing.T) {
|
||||||
|
state, err := load(mapping("prompt resources", map[string]any{"prompts": []any{
|
||||||
|
map[string]any{"id": "base", "default_profile": "missing", "inputs": []any{map[string]any{"name": "x", "required": false}, map[string]any{"name": "y"}},
|
||||||
|
"messages": []any{map[string]any{"role": "user", "content": "", "cache_control": map[string]any{"type": "ephemeral", "ttl": ""}}, map[string]any{"role": "assistant", "content": "{{.missing}}", "cache_control": map[string]any{"type": "ephemeral"}}},
|
||||||
|
"output": map[string]any{"format": "text", "validation_mode": "basic", "repair_attempts": 0}},
|
||||||
|
map[string]any{"id": "child", "extends": map[string]any{"id": "base"}},
|
||||||
|
}}), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := state.InspectPrompt(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.DefaultProfile == nil || *got.DefaultProfile != "missing" || got.Inputs == nil || (*got.Inputs)[0].Required == nil || *(*got.Inputs)[0].Required || (*got.Inputs)[1].Required != nil || got.Output.RepairAttempts == nil || *got.Output.RepairAttempts != 0 || got.Messages[0].Content != "" || got.Messages[0].CacheControl.TTL == nil || *got.Messages[0].CacheControl.TTL != "" || got.Messages[1].CacheControl.TTL != nil {
|
||||||
|
t.Fatalf("prompt resource = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
func TestLoadHeaderlessImportedItem(t *testing.T) {
|
||||||
files := fstest.MapFS{
|
files := fstest.MapFS{
|
||||||
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
"main.yaml": {Data: []byte("version: 2\nconfig:\n imports:\n - path: item.yaml\n destination: [prompts]\n placement: sequence-item\n required: true\n")},
|
||||||
|
|||||||
@@ -32,6 +32,74 @@ type BackendResource struct {
|
|||||||
SmokeTests *domain.BackendSmokeTestConfig
|
SmokeTests *domain.BackendSmokeTestConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PromptResource is the resolved prompt with already captured message text.
|
||||||
|
type PromptResource struct {
|
||||||
|
ID, Hash string
|
||||||
|
DefaultProfile, Description, SessionID *string
|
||||||
|
Inputs *[]PromptInputResource
|
||||||
|
Messages []PromptMessageResource
|
||||||
|
Output PromptOutputResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptInputResource struct {
|
||||||
|
Name string
|
||||||
|
Required *bool
|
||||||
|
ContentType, Description *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptMessageResource struct {
|
||||||
|
Role, Content string
|
||||||
|
CacheControl *PromptCacheControlResource
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptCacheControlResource struct {
|
||||||
|
Type domain.CacheControlType
|
||||||
|
TTL *string
|
||||||
|
}
|
||||||
|
|
||||||
|
type PromptOutputResource struct {
|
||||||
|
Format domain.OutputFormat
|
||||||
|
ValidationMode domain.ValidationMode
|
||||||
|
SchemaPath *string
|
||||||
|
RepairAttempts *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (state *loaded) InspectPrompt(ctx context.Context, id string) (*PromptResource, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entry, err := state.lookupPromptEntry(id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, promptLookupError(err)
|
||||||
|
}
|
||||||
|
raw := entry.resource
|
||||||
|
result := &PromptResource{ID: id, Hash: entry.hash, DefaultProfile: raw.DefaultProfile, Description: raw.Description, SessionID: raw.SessionID}
|
||||||
|
if raw.Inputs != nil {
|
||||||
|
inputs := make([]PromptInputResource, len(raw.Inputs))
|
||||||
|
for i, input := range raw.Inputs {
|
||||||
|
inputs[i] = PromptInputResource{Name: input.Name, ContentType: input.ContentType, Description: input.Description}
|
||||||
|
if state.promptMetadata[id][i].required == fieldValue {
|
||||||
|
required := input.Required
|
||||||
|
inputs[i].Required = &required
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Inputs = &inputs
|
||||||
|
}
|
||||||
|
result.Messages = make([]PromptMessageResource, len(entry.definition.Templates))
|
||||||
|
for i, message := range entry.definition.Templates {
|
||||||
|
result.Messages[i] = PromptMessageResource{Role: message.Role, Content: message.Content}
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
result.Messages[i].CacheControl = &PromptCacheControlResource{Type: message.CacheControl.Type, TTL: raw.Messages[i].CacheControl.TTL}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.Output = PromptOutputResource{Format: entry.definition.Validation.Format, ValidationMode: entry.definition.Validation.ValidationMode, SchemaPath: raw.Output.SchemaPath}
|
||||||
|
if state.promptRepair[id] {
|
||||||
|
value := raw.Output.RepairAttempts
|
||||||
|
result.Output.RepairAttempts = &value
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (state *loaded) InspectProfile(ctx context.Context, id string) (*ProfileResource, error) {
|
func (state *loaded) InspectProfile(ctx context.Context, id string) (*ProfileResource, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ type PromptDefinition struct {
|
|||||||
Validation OutputContract `yaml:"validation"`
|
Validation OutputContract `yaml:"validation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PromptInspection is the resolved result of exact prompt inspection.
|
// PromptInspection is lightweight metadata for prompt requirement selection.
|
||||||
type PromptInspection struct {
|
type PromptInspection struct {
|
||||||
PromptID string
|
PromptID string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|||||||
// Repository selects prompt definitions from a captured index.
|
// Repository selects prompt definitions from a captured index.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
GetPromptSnapshot(ctx context.Context, id string) (*domain.PromptDefinition, string, error)
|
||||||
|
// GetPromptInspection supplies lightweight requirement-selection metadata.
|
||||||
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
GetPromptInspection(ctx context.Context, id string) (*domain.PromptInspection, error)
|
||||||
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
GetRequirementSnapshot(id string) (prompt.RequirementSnapshot, bool)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user