Add detailed comparison assertions
This commit is contained in:
13
docs/cli.md
13
docs/cli.md
@@ -511,8 +511,21 @@ example config diff --config before.yml --against-config after.yml
|
|||||||
example config diff --config base.yml --against-overlay candidate
|
example config diff --config base.yml --against-overlay candidate
|
||||||
example config diff --config base.yml --overlay current --against-overlay ""
|
example config diff --config base.yml --overlay current --against-overlay ""
|
||||||
example config diff --scope promptkit --config before.yml --against-config after.yml
|
example config diff --scope promptkit --config before.yml --against-config after.yml
|
||||||
|
example config diff --config before.yml --against-config after.yml --details --check
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`--details` replaces the concise report with `{scope, differences}`, containing
|
||||||
|
one entry per native difference. Each entry has `path`, `before`, `after`, and
|
||||||
|
`hidden_changed`; each snapshot has `present`, `redacted`, and `value`. An
|
||||||
|
absent snapshot has `present: false` and a null value, distinct from a present
|
||||||
|
null. Empty comparisons contain `differences: []`.
|
||||||
|
|
||||||
|
`--check` returns status `3` when native differences exist and `0` when they do
|
||||||
|
not. Without it, a completed comparison succeeds even when differences exist.
|
||||||
|
Usage errors retain status `2`, and operational failures retain status `1`.
|
||||||
|
The report is still delivered in check mode; a presentation failure remains
|
||||||
|
separate evidence and does not replace a nonzero difference outcome.
|
||||||
|
|
||||||
For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values
|
For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values
|
||||||
after defaults and resolution; source or provenance differences alone do not
|
after defaults and resolution; source or provenance differences alone do not
|
||||||
create a change. PromptKit delegates its native comparison to ConfigKit. The
|
create a change. PromptKit delegates its native comparison to ConfigKit. The
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Shared reporting and PromptKit v0.20.0 implementation plan
|
# Shared reporting and PromptKit v0.20.0 implementation plan
|
||||||
|
|
||||||
Status: implementation in progress; Stages 1–18 complete.
|
Status: implementation in progress; Stages 1–19 complete.
|
||||||
|
|
||||||
## Scope, authority, and execution
|
## Scope, authority, and execution
|
||||||
|
|
||||||
@@ -680,6 +680,10 @@ source files again to recover evidence.
|
|||||||
|
|
||||||
## Stage 19: Detailed differences and assertion outcomes
|
## Stage 19: Detailed differences and assertion outcomes
|
||||||
|
|
||||||
|
**Complete.** Detailed native difference reports and opt-in assertion status
|
||||||
|
are implemented. Root, ckconfig, and ckprompt workspace suites passed with Go
|
||||||
|
1.27.0, including concealed changes, null versus absence, and delivery failures.
|
||||||
|
|
||||||
Reuse existing Scope.Compare and native Difference records. Keep concise
|
Reuse existing Scope.Compare and native Difference records. Keep concise
|
||||||
additions/changes/deletions as default and add `--details`. The detailed report
|
additions/changes/deletions as default and add `--details`. The detailed report
|
||||||
is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`,
|
is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`,
|
||||||
|
|||||||
104
integrations/ckconfig/detailed_diff_test.go
Normal file
104
integrations/ckconfig/detailed_diff_test.go
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
package ckconfig
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
configkit "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDetailedDiffPreservesNativeEvidenceAndAssertions(t *testing.T) {
|
||||||
|
values := []configkit.Difference{
|
||||||
|
{Path: configkit.Keys("added"), Right: configkit.ValueSnapshot{Present: true, YAML: "null\n"}},
|
||||||
|
{Path: configkit.Keys("removed"), Left: configkit.ValueSnapshot{Present: true, YAML: "false\n"}},
|
||||||
|
{Path: configkit.Keys("secret"), Left: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, Right: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, HiddenChanged: true},
|
||||||
|
{Path: configkit.Keys("list"), Left: configkit.ValueSnapshot{Present: true, YAML: "[one, two]\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "[three]\n"}},
|
||||||
|
{Path: configkit.Keys("marker"), Left: configkit.ValueSnapshot{Present: true, YAML: "'redacted'\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "'***'\n"}},
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||||
|
calls++
|
||||||
|
return commandkit.Acquired[[]configkit.Difference]{Value: values}, nil
|
||||||
|
}}
|
||||||
|
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||||
|
for _, format := range []string{"json", "yaml", "text"} {
|
||||||
|
var out bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||||
|
var logical any
|
||||||
|
var err error
|
||||||
|
if format == "json" {
|
||||||
|
err = json.Unmarshal(out.Bytes(), &logical)
|
||||||
|
} else {
|
||||||
|
logical, err = commandkit.DecodeYAMLReport(&out)
|
||||||
|
}
|
||||||
|
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || err != nil {
|
||||||
|
t.Fatalf("format %s = %#v %v %q", format, result, err, out.String())
|
||||||
|
}
|
||||||
|
report := logical.(map[string]any)
|
||||||
|
entries := report["differences"].([]any)
|
||||||
|
if report["scope"] != "app" || len(entries) != len(values) {
|
||||||
|
t.Fatalf("format %s report %#v", format, report)
|
||||||
|
}
|
||||||
|
added := entries[0].(map[string]any)
|
||||||
|
before, after := added["before"].(map[string]any), added["after"].(map[string]any)
|
||||||
|
if before["present"] != false || before["value"] != nil || after["present"] != true || after["value"] != nil {
|
||||||
|
t.Fatalf("null/absence: %#v", added)
|
||||||
|
}
|
||||||
|
secret := entries[2].(map[string]any)
|
||||||
|
if secret["hidden_changed"] != true || !reflect.DeepEqual(secret["before"].(map[string]any)["value"], secret["after"].(map[string]any)["value"]) {
|
||||||
|
t.Fatalf("hidden change: %#v", secret)
|
||||||
|
}
|
||||||
|
list := entries[3].(map[string]any)["after"].(map[string]any)["value"]
|
||||||
|
if !reflect.DeepEqual(list, []any{"three"}) {
|
||||||
|
t.Fatalf("atomic sequence: %#v", list)
|
||||||
|
}
|
||||||
|
marker := entries[4].(map[string]any)["after"].(map[string]any)
|
||||||
|
if marker["value"] != "***" || marker["redacted"] != false {
|
||||||
|
t.Fatalf("marker string: %#v", marker)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls != 3 {
|
||||||
|
t.Fatalf("provider calls=%d", calls)
|
||||||
|
}
|
||||||
|
result := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeSuccess || calls != 4 {
|
||||||
|
t.Fatalf("ordinary diff: %#v calls=%d", result, calls)
|
||||||
|
}
|
||||||
|
values = nil
|
||||||
|
var out bytes.Buffer
|
||||||
|
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeSuccess || !strings.Contains(out.String(), `"differences": []`) {
|
||||||
|
t.Fatalf("equality: %#v %q", result, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDiffAssertionRetainsDomainOutcomeOnDeliveryFailures(t *testing.T) {
|
||||||
|
values := []configkit.Difference{{Path: configkit.Keys("x"), Right: configkit.ValueSnapshot{Present: true, YAML: "true\n"}}}
|
||||||
|
cause := errors.New("provider failed")
|
||||||
|
var providerErr error
|
||||||
|
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||||
|
return commandkit.Acquired[[]configkit.Difference]{Value: values}, providerErr
|
||||||
|
}}
|
||||||
|
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||||
|
result := app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
|
||||||
|
t.Fatalf("writer failure: %#v", result)
|
||||||
|
}
|
||||||
|
values[0].Right.YAML = "[invalid"
|
||||||
|
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.RenderError == nil {
|
||||||
|
t.Fatalf("render failure: %#v", result)
|
||||||
|
}
|
||||||
|
providerErr = cause
|
||||||
|
result = app.Execute(context.Background(), []string{"config", "diff", "--check"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||||
|
if result.Kind != commandkit.OutcomeOperational || !errors.Is(result.Cause, cause) {
|
||||||
|
t.Fatalf("provider failure: %#v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,7 +30,14 @@ func runConfigDiff[I, S any](invocation commandkit.Invocation, plan *configOpera
|
|||||||
if err := invocation.Context.Err(); err != nil {
|
if err := invocation.Context.Err(); err != nil {
|
||||||
return configFailure("Configuration comparison was canceled.", acquired.Explanation, err)
|
return configFailure("Configuration comparison was canceled.", acquired.Explanation, err)
|
||||||
}
|
}
|
||||||
return renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
|
outcome := renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
|
||||||
|
if plan.details {
|
||||||
|
return renderDetailedDiff(selected.scope.scope.Name, acquired.Value)
|
||||||
|
}
|
||||||
return renderDiffReport(acquired.Value)
|
return renderDiffReport(acquired.Value)
|
||||||
})
|
})
|
||||||
|
if outcome.Kind == commandkit.OutcomeSuccess && plan.check && len(acquired.Value) > 0 {
|
||||||
|
return commandkit.Domain("configuration differences", 3)
|
||||||
|
}
|
||||||
|
return outcome
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,6 +78,52 @@ type diffReport struct {
|
|||||||
Deletions []string `json:"deletions"`
|
Deletions []string `json:"deletions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type detailedDiffReport struct {
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Differences []detailedDifference `json:"differences"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type detailedDifference struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
Before detailedDiffSnapshot `json:"before"`
|
||||||
|
After detailedDiffSnapshot `json:"after"`
|
||||||
|
HiddenChanged bool `json:"hidden_changed"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type detailedDiffSnapshot struct {
|
||||||
|
Present bool `json:"present"`
|
||||||
|
Redacted bool `json:"redacted"`
|
||||||
|
Value any `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderDetailedDiff(scope string, differences []configkit.Difference) (any, error) {
|
||||||
|
report := detailedDiffReport{Scope: scope, Differences: make([]detailedDifference, 0, len(differences))}
|
||||||
|
for _, difference := range differences {
|
||||||
|
before, err := detailedSnapshot(difference.Left)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("difference at %q before: %w", difference.Path.String(), err)
|
||||||
|
}
|
||||||
|
after, err := detailedSnapshot(difference.Right)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("difference at %q after: %w", difference.Path.String(), err)
|
||||||
|
}
|
||||||
|
report.Differences = append(report.Differences, detailedDifference{Path: difference.Path.String(), Before: before, After: after, HiddenChanged: difference.HiddenChanged})
|
||||||
|
}
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func detailedSnapshot(snapshot configkit.ValueSnapshot) (detailedDiffSnapshot, error) {
|
||||||
|
report := detailedDiffSnapshot{Present: snapshot.Present, Redacted: snapshot.Redacted}
|
||||||
|
if snapshot.Present {
|
||||||
|
value, err := commandkit.DecodeYAMLReport(strings.NewReader(snapshot.YAML))
|
||||||
|
if err != nil {
|
||||||
|
return report, err
|
||||||
|
}
|
||||||
|
report.Value = value
|
||||||
|
}
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
|
||||||
func renderDiffReport(differences []configkit.Difference) (any, error) {
|
func renderDiffReport(differences []configkit.Difference) (any, error) {
|
||||||
report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}}
|
report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}}
|
||||||
for _, difference := range differences {
|
for _, difference := range differences {
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ type configCommandInput[I, S any] struct {
|
|||||||
under commandkit.Supplied[string]
|
under commandkit.Supplied[string]
|
||||||
explicit commandkit.Supplied[bool]
|
explicit commandkit.Supplied[bool]
|
||||||
sourceID commandkit.Supplied[string]
|
sourceID commandkit.Supplied[string]
|
||||||
|
details commandkit.Supplied[bool]
|
||||||
|
check commandkit.Supplied[bool]
|
||||||
values commandkit.FlagValues[I]
|
values commandkit.FlagValues[I]
|
||||||
againstValues commandkit.FlagValues[I]
|
againstValues commandkit.FlagValues[I]
|
||||||
plan *configOperationPlan[I, S]
|
plan *configOperationPlan[I, S]
|
||||||
@@ -41,6 +43,8 @@ type configOperationPlan[I, S any] struct {
|
|||||||
path configkit.Path
|
path configkit.Path
|
||||||
pathSet bool
|
pathSet bool
|
||||||
sourceID commandkit.Supplied[string]
|
sourceID commandkit.Supplied[string]
|
||||||
|
details bool
|
||||||
|
check bool
|
||||||
list bool
|
list bool
|
||||||
help bool
|
help bool
|
||||||
historical bool
|
historical bool
|
||||||
@@ -78,6 +82,8 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
|
|||||||
}
|
}
|
||||||
if operation == Diff {
|
if operation == Diff {
|
||||||
flags = append(flags,
|
flags = append(flags,
|
||||||
|
commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native before and after difference evidence."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
|
||||||
|
commandkit.BoolFlag(commandkit.FlagMeta{Name: "check", Usage: "Return status 3 when differences are found."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).check = value }),
|
||||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString,
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString,
|
||||||
func(input **configCommandInput[I, S], value commandkit.Supplied[string]) {
|
func(input **configCommandInput[I, S], value commandkit.Supplied[string]) {
|
||||||
(*input).against.Config = value
|
(*input).against.Config = value
|
||||||
@@ -248,6 +254,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
|
|||||||
path: path,
|
path: path,
|
||||||
pathSet: input.path.Set,
|
pathSet: input.path.Set,
|
||||||
sourceID: input.sourceID,
|
sourceID: input.sourceID,
|
||||||
|
details: input.details.Value,
|
||||||
|
check: input.check.Value,
|
||||||
list: input.list.Value,
|
list: input.list.Value,
|
||||||
historical: input.historical.Value,
|
historical: input.historical.Value,
|
||||||
under: under,
|
under: under,
|
||||||
|
|||||||
Reference in New Issue
Block a user