Add PromptKit requirements YAML report
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
# Implement PromptKit requirements inspection
|
# Implement PromptKit requirements inspection
|
||||||
|
|
||||||
Status: in progress; Stage 1 is complete.
|
Status: in progress; Stages 1 and 2 are complete.
|
||||||
|
|
||||||
## Authority and working rules
|
## Authority and working rules
|
||||||
|
|
||||||
@@ -167,6 +167,14 @@ Run the focused report tests. Use only the module's declared
|
|||||||
`go.yaml.in/yaml/v4` package in new tests. Stage 3 will connect this report to
|
`go.yaml.in/yaml/v4` package in new tests. Stage 3 will connect this report to
|
||||||
invocation and delivery; do not create a parallel public report writer.
|
invocation and delivery; do not create a parallel public report writer.
|
||||||
|
|
||||||
|
**Completed:** Added private requirements report types and a direct conversion
|
||||||
|
from PromptKit's native inspection in `integrations/ckprompt/command.go`, with
|
||||||
|
semantic YAML coverage in `command_test.go`. `GOTOOLCHAIN=local go test
|
||||||
|
./integrations/ckprompt -run 'TestRequirementsReport' -count=1` passed. Coverage
|
||||||
|
confirms empty sequences, present false/empty values, nullable metadata,
|
||||||
|
ordering, unrecognized kind/code strings, and complete and partial results.
|
||||||
|
No material limitations were encountered.
|
||||||
|
|
||||||
## Stage 3: Assemble and invoke the optional requirements command
|
## Stage 3: Assemble and invoke the optional requirements command
|
||||||
|
|
||||||
**Outcome:** an installed requirements capability has the selected command,
|
**Outcome:** an installed requirements capability has the selected command,
|
||||||
|
|||||||
@@ -316,6 +316,39 @@ type promptInspectionReport struct {
|
|||||||
OutputContract promptOutputContractReport `yaml:"output_contract"`
|
OutputContract promptOutputContractReport `yaml:"output_contract"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type requirementsInspectionReport struct {
|
||||||
|
PromptID string `yaml:"prompt_id"`
|
||||||
|
PromptVersion string `yaml:"prompt_version"`
|
||||||
|
ProfileID string `yaml:"profile_id"`
|
||||||
|
Complete bool `yaml:"complete"`
|
||||||
|
Inputs []requirementInputReport `yaml:"inputs"`
|
||||||
|
Variables []requirementVariableReport `yaml:"variables"`
|
||||||
|
Issues []requirementIssueReport `yaml:"issues"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementInputReport struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Kind string `yaml:"kind"`
|
||||||
|
Declared bool `yaml:"declared"`
|
||||||
|
ContentType *string `yaml:"content_type"`
|
||||||
|
Description *string `yaml:"description"`
|
||||||
|
Explanation string `yaml:"explanation"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementVariableReport struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Kind string `yaml:"kind"`
|
||||||
|
Supplied bool `yaml:"supplied"`
|
||||||
|
Explanation string `yaml:"explanation"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type requirementIssueReport struct {
|
||||||
|
Code string `yaml:"code"`
|
||||||
|
InputName string `yaml:"input_name"`
|
||||||
|
VariableName string `yaml:"variable_name"`
|
||||||
|
Description string `yaml:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
type modelParamsReport struct {
|
type modelParamsReport struct {
|
||||||
BackendID string `yaml:"backend_id"`
|
BackendID string `yaml:"backend_id"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
@@ -356,6 +389,45 @@ func promptReportFromInspection(inspection *promptkit.PromptInspection) promptIn
|
|||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func requirementsReportFromInspection(inspection *promptkit.RequirementInspection) requirementsInspectionReport {
|
||||||
|
report := requirementsInspectionReport{
|
||||||
|
PromptID: inspection.PromptID,
|
||||||
|
PromptVersion: inspection.PromptVersion,
|
||||||
|
ProfileID: inspection.ProfileID,
|
||||||
|
Complete: inspection.Complete,
|
||||||
|
Inputs: make([]requirementInputReport, 0, len(inspection.Inputs)),
|
||||||
|
Variables: make([]requirementVariableReport, 0, len(inspection.Variables)),
|
||||||
|
Issues: make([]requirementIssueReport, 0, len(inspection.Issues)),
|
||||||
|
}
|
||||||
|
for _, input := range inspection.Inputs {
|
||||||
|
report.Inputs = append(report.Inputs, requirementInputReport{
|
||||||
|
Name: input.Name,
|
||||||
|
Kind: string(input.Kind),
|
||||||
|
Declared: input.Declared,
|
||||||
|
ContentType: input.ContentType,
|
||||||
|
Description: input.Description,
|
||||||
|
Explanation: input.Explanation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, variable := range inspection.Variables {
|
||||||
|
report.Variables = append(report.Variables, requirementVariableReport{
|
||||||
|
Name: variable.Name,
|
||||||
|
Kind: string(variable.Kind),
|
||||||
|
Supplied: variable.Supplied,
|
||||||
|
Explanation: variable.Explanation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, issue := range inspection.Issues {
|
||||||
|
report.Issues = append(report.Issues, requirementIssueReport{
|
||||||
|
Code: string(issue.Code),
|
||||||
|
InputName: issue.InputName,
|
||||||
|
VariableName: issue.VariableName,
|
||||||
|
Description: issue.Description,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return report
|
||||||
|
}
|
||||||
|
|
||||||
func profileReportFromInspection(inspection *promptkit.ProfileInspection) profileInspectionReport {
|
func profileReportFromInspection(inspection *promptkit.ProfileInspection) profileInspectionReport {
|
||||||
params := inspection.EffectiveModelParams
|
params := inspection.EffectiveModelParams
|
||||||
extra := params.ExtraParams
|
extra := params.ExtraParams
|
||||||
|
|||||||
@@ -748,3 +748,110 @@ func TestProfileCatalogYAMLPreservesStringValuesAndOrder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRequirementsReportRendersEmptyInventoriesAndFalseValues(t *testing.T) {
|
||||||
|
output, err := encodePromptYAMLBytes(requirementsReportFromInspection(&promptkit.RequirementInspection{}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var report map[string]any
|
||||||
|
if err := yaml.Unmarshal(output, &report); err != nil {
|
||||||
|
t.Fatalf("decode requirements report %q: %v", output, err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"prompt_id", "prompt_version", "profile_id", "complete", "inputs", "variables", "issues"} {
|
||||||
|
if _, ok := report[key]; !ok {
|
||||||
|
t.Errorf("requirements report is missing %q: %q", key, output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if report["complete"] != false {
|
||||||
|
t.Errorf("complete = %#v, want false", report["complete"])
|
||||||
|
}
|
||||||
|
for _, key := range []string{"inputs", "variables", "issues"} {
|
||||||
|
items, ok := report[key].([]any)
|
||||||
|
if !ok || len(items) != 0 {
|
||||||
|
t.Errorf("%s = %#v, want an empty sequence", key, report[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range []string{"prompt_id", "prompt_version", "profile_id"} {
|
||||||
|
if report[key] != "" {
|
||||||
|
t.Errorf("%s = %#v, want an explicitly present empty string", key, report[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequirementsReportPreservesNativeFieldsAndMetadata(t *testing.T) {
|
||||||
|
empty := ""
|
||||||
|
contentType := "true"
|
||||||
|
inspection := &promptkit.RequirementInspection{
|
||||||
|
PromptID: "true",
|
||||||
|
PromptVersion: "001",
|
||||||
|
ProfileID: "null",
|
||||||
|
Complete: true,
|
||||||
|
Inputs: []promptkit.InputRequirement{
|
||||||
|
{Name: "true", Kind: promptkit.RequirementKind("123"), Declared: false, Description: &empty},
|
||||||
|
{Name: "001", Kind: promptkit.RequirementKind("future_kind"), Declared: true, ContentType: &contentType, Explanation: "false"},
|
||||||
|
},
|
||||||
|
Variables: []promptkit.VariableRequirement{
|
||||||
|
{Name: "false", Kind: promptkit.RequirementKind("on"), Supplied: false},
|
||||||
|
{Name: "001", Kind: promptkit.RequirementKind("future_kind"), Supplied: true, Explanation: "true"},
|
||||||
|
},
|
||||||
|
Issues: []promptkit.RequirementIssue{{
|
||||||
|
Code: promptkit.RequirementIssueCode("123"),
|
||||||
|
InputName: "",
|
||||||
|
VariableName: "false",
|
||||||
|
Description: "",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, complete := range []bool{true, false} {
|
||||||
|
t.Run(map[bool]string{true: "complete", false: "partial"}[complete], func(t *testing.T) {
|
||||||
|
inspection.Complete = complete
|
||||||
|
output, err := encodePromptYAMLBytes(requirementsReportFromInspection(inspection))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var report map[string]any
|
||||||
|
if err := yaml.Unmarshal(output, &report); err != nil {
|
||||||
|
t.Fatalf("decode requirements report %q: %v", output, err)
|
||||||
|
}
|
||||||
|
if report["complete"] != complete || report["prompt_id"] != "true" || report["prompt_version"] != "001" || report["profile_id"] != "null" {
|
||||||
|
t.Fatalf("selected context or completeness = %#v", report)
|
||||||
|
}
|
||||||
|
|
||||||
|
inputs, ok := report["inputs"].([]any)
|
||||||
|
if !ok || len(inputs) != 2 {
|
||||||
|
t.Fatalf("inputs = %#v", report["inputs"])
|
||||||
|
}
|
||||||
|
firstInput, ok := inputs[0].(map[string]any)
|
||||||
|
if !ok || len(firstInput) != 6 || firstInput["name"] != "true" || firstInput["kind"] != "123" || firstInput["declared"] != false || firstInput["content_type"] != nil || firstInput["description"] != "" || firstInput["explanation"] != "" {
|
||||||
|
t.Fatalf("first input = %#v", inputs[0])
|
||||||
|
}
|
||||||
|
secondInput, ok := inputs[1].(map[string]any)
|
||||||
|
if !ok || len(secondInput) != 6 || secondInput["name"] != "001" || secondInput["kind"] != "future_kind" || secondInput["declared"] != true || secondInput["content_type"] != "true" || secondInput["description"] != nil || secondInput["explanation"] != "false" {
|
||||||
|
t.Fatalf("second input = %#v", inputs[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
variables, ok := report["variables"].([]any)
|
||||||
|
if !ok || len(variables) != 2 {
|
||||||
|
t.Fatalf("variables = %#v", report["variables"])
|
||||||
|
}
|
||||||
|
firstVariable, ok := variables[0].(map[string]any)
|
||||||
|
if !ok || len(firstVariable) != 4 || firstVariable["name"] != "false" || firstVariable["kind"] != "on" || firstVariable["supplied"] != false || firstVariable["explanation"] != "" {
|
||||||
|
t.Fatalf("first variable = %#v", variables[0])
|
||||||
|
}
|
||||||
|
secondVariable, ok := variables[1].(map[string]any)
|
||||||
|
if !ok || len(secondVariable) != 4 || secondVariable["name"] != "001" || secondVariable["kind"] != "future_kind" || secondVariable["supplied"] != true || secondVariable["explanation"] != "true" {
|
||||||
|
t.Fatalf("second variable = %#v", variables[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
issues, ok := report["issues"].([]any)
|
||||||
|
if !ok || len(issues) != 1 {
|
||||||
|
t.Fatalf("issues = %#v", report["issues"])
|
||||||
|
}
|
||||||
|
issue, ok := issues[0].(map[string]any)
|
||||||
|
if !ok || len(issue) != 4 || issue["code"] != "123" || issue["input_name"] != "" || issue["variable_name"] != "false" || issue["description"] != "" {
|
||||||
|
t.Fatalf("issue = %#v", issues[0])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user