Add detailed comparison assertions

This commit is contained in:
2026-11-07 04:49:40 +00:00
parent 65b454b2ab
commit 1986c44f49
6 changed files with 184 additions and 2 deletions

View File

@@ -511,8 +511,21 @@ example config diff --config before.yml --against-config after.yml
example config diff --config base.yml --against-overlay candidate example config diff --config base.yml --against-overlay candidate
example config diff --config base.yml --overlay current --against-overlay "" example config diff --config base.yml --overlay current --against-overlay ""
example config diff --scope promptkit --config before.yml --against-config after.yml example config diff --scope promptkit --config before.yml --against-config after.yml
example config diff --config before.yml --against-config after.yml --details --check
``` ```
`--details` replaces the concise report with `{scope, differences}`, containing
one entry per native difference. Each entry has `path`, `before`, `after`, and
`hidden_changed`; each snapshot has `present`, `redacted`, and `value`. An
absent snapshot has `present: false` and a null value, distinct from a present
null. Empty comparisons contain `differences: []`.
`--check` returns status `3` when native differences exist and `0` when they do
not. Without it, a completed comparison succeeds even when differences exist.
Usage errors retain status `2`, and operational failures retain status `1`.
The report is still delivered in check mode; a presentation failure remains
separate evidence and does not replace a nonzero difference outcome.
For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values
after defaults and resolution; source or provenance differences alone do not after defaults and resolution; source or provenance differences alone do not
create a change. PromptKit delegates its native comparison to ConfigKit. The create a change. PromptKit delegates its native comparison to ConfigKit. The

View File

@@ -1,6 +1,6 @@
# Shared reporting and PromptKit v0.20.0 implementation plan # Shared reporting and PromptKit v0.20.0 implementation plan
Status: implementation in progress; Stages 1–18 complete. Status: implementation in progress; Stages 1–19 complete.
## Scope, authority, and execution ## Scope, authority, and execution
@@ -680,6 +680,10 @@ source files again to recover evidence.
## Stage 19: Detailed differences and assertion outcomes ## Stage 19: Detailed differences and assertion outcomes
**Complete.** Detailed native difference reports and opt-in assertion status
are implemented. Root, ckconfig, and ckprompt workspace suites passed with Go
1.27.0, including concealed changes, null versus absence, and delivery failures.
Reuse existing Scope.Compare and native Difference records. Keep concise Reuse existing Scope.Compare and native Difference records. Keep concise
additions/changes/deletions as default and add `--details`. The detailed report additions/changes/deletions as default and add `--details`. The detailed report
is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`, is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`,

View File

@@ -0,0 +1,104 @@
package ckconfig
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"reflect"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
configkit "go.maximumdirect.net/configkit"
)
func TestDetailedDiffPreservesNativeEvidenceAndAssertions(t *testing.T) {
values := []configkit.Difference{
{Path: configkit.Keys("added"), Right: configkit.ValueSnapshot{Present: true, YAML: "null\n"}},
{Path: configkit.Keys("removed"), Left: configkit.ValueSnapshot{Present: true, YAML: "false\n"}},
{Path: configkit.Keys("secret"), Left: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, Right: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, HiddenChanged: true},
{Path: configkit.Keys("list"), Left: configkit.ValueSnapshot{Present: true, YAML: "[one, two]\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "[three]\n"}},
{Path: configkit.Keys("marker"), Left: configkit.ValueSnapshot{Present: true, YAML: "'redacted'\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "'***'\n"}},
}
calls := 0
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
calls++
return commandkit.Acquired[[]configkit.Difference]{Value: values}, nil
}}
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
for _, format := range []string{"json", "yaml", "text"} {
var out bytes.Buffer
result := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
var logical any
var err error
if format == "json" {
err = json.Unmarshal(out.Bytes(), &logical)
} else {
logical, err = commandkit.DecodeYAMLReport(&out)
}
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || err != nil {
t.Fatalf("format %s = %#v %v %q", format, result, err, out.String())
}
report := logical.(map[string]any)
entries := report["differences"].([]any)
if report["scope"] != "app" || len(entries) != len(values) {
t.Fatalf("format %s report %#v", format, report)
}
added := entries[0].(map[string]any)
before, after := added["before"].(map[string]any), added["after"].(map[string]any)
if before["present"] != false || before["value"] != nil || after["present"] != true || after["value"] != nil {
t.Fatalf("null/absence: %#v", added)
}
secret := entries[2].(map[string]any)
if secret["hidden_changed"] != true || !reflect.DeepEqual(secret["before"].(map[string]any)["value"], secret["after"].(map[string]any)["value"]) {
t.Fatalf("hidden change: %#v", secret)
}
list := entries[3].(map[string]any)["after"].(map[string]any)["value"]
if !reflect.DeepEqual(list, []any{"three"}) {
t.Fatalf("atomic sequence: %#v", list)
}
marker := entries[4].(map[string]any)["after"].(map[string]any)
if marker["value"] != "***" || marker["redacted"] != false {
t.Fatalf("marker string: %#v", marker)
}
}
if calls != 3 {
t.Fatalf("provider calls=%d", calls)
}
result := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeSuccess || calls != 4 {
t.Fatalf("ordinary diff: %#v calls=%d", result, calls)
}
values = nil
var out bytes.Buffer
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Kind != commandkit.OutcomeSuccess || !strings.Contains(out.String(), `"differences": []`) {
t.Fatalf("equality: %#v %q", result, out.String())
}
}
func TestDiffAssertionRetainsDomainOutcomeOnDeliveryFailures(t *testing.T) {
values := []configkit.Difference{{Path: configkit.Keys("x"), Right: configkit.ValueSnapshot{Present: true, YAML: "true\n"}}}
cause := errors.New("provider failed")
var providerErr error
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
return commandkit.Acquired[[]configkit.Difference]{Value: values}, providerErr
}}
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
result := app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
t.Fatalf("writer failure: %#v", result)
}
values[0].Right.YAML = "[invalid"
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.RenderError == nil {
t.Fatalf("render failure: %#v", result)
}
providerErr = cause
result = app.Execute(context.Background(), []string{"config", "diff", "--check"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeOperational || !errors.Is(result.Cause, cause) {
t.Fatalf("provider failure: %#v", result)
}
}

View File

@@ -30,7 +30,14 @@ func runConfigDiff[I, S any](invocation commandkit.Invocation, plan *configOpera
if err := invocation.Context.Err(); err != nil { if err := invocation.Context.Err(); err != nil {
return configFailure("Configuration comparison was canceled.", acquired.Explanation, err) return configFailure("Configuration comparison was canceled.", acquired.Explanation, err)
} }
return renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) { outcome := renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
if plan.details {
return renderDetailedDiff(selected.scope.scope.Name, acquired.Value)
}
return renderDiffReport(acquired.Value) return renderDiffReport(acquired.Value)
}) })
if outcome.Kind == commandkit.OutcomeSuccess && plan.check && len(acquired.Value) > 0 {
return commandkit.Domain("configuration differences", 3)
}
return outcome
} }

View File

@@ -78,6 +78,52 @@ type diffReport struct {
Deletions []string `json:"deletions"` Deletions []string `json:"deletions"`
} }
type detailedDiffReport struct {
Scope string `json:"scope"`
Differences []detailedDifference `json:"differences"`
}
type detailedDifference struct {
Path string `json:"path"`
Before detailedDiffSnapshot `json:"before"`
After detailedDiffSnapshot `json:"after"`
HiddenChanged bool `json:"hidden_changed"`
}
type detailedDiffSnapshot struct {
Present bool `json:"present"`
Redacted bool `json:"redacted"`
Value any `json:"value"`
}
func renderDetailedDiff(scope string, differences []configkit.Difference) (any, error) {
report := detailedDiffReport{Scope: scope, Differences: make([]detailedDifference, 0, len(differences))}
for _, difference := range differences {
before, err := detailedSnapshot(difference.Left)
if err != nil {
return nil, fmt.Errorf("difference at %q before: %w", difference.Path.String(), err)
}
after, err := detailedSnapshot(difference.Right)
if err != nil {
return nil, fmt.Errorf("difference at %q after: %w", difference.Path.String(), err)
}
report.Differences = append(report.Differences, detailedDifference{Path: difference.Path.String(), Before: before, After: after, HiddenChanged: difference.HiddenChanged})
}
return report, nil
}
func detailedSnapshot(snapshot configkit.ValueSnapshot) (detailedDiffSnapshot, error) {
report := detailedDiffSnapshot{Present: snapshot.Present, Redacted: snapshot.Redacted}
if snapshot.Present {
value, err := commandkit.DecodeYAMLReport(strings.NewReader(snapshot.YAML))
if err != nil {
return report, err
}
report.Value = value
}
return report, nil
}
func renderDiffReport(differences []configkit.Difference) (any, error) { func renderDiffReport(differences []configkit.Difference) (any, error) {
report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}} report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}}
for _, difference := range differences { for _, difference := range differences {

View File

@@ -18,6 +18,8 @@ type configCommandInput[I, S any] struct {
under commandkit.Supplied[string] under commandkit.Supplied[string]
explicit commandkit.Supplied[bool] explicit commandkit.Supplied[bool]
sourceID commandkit.Supplied[string] sourceID commandkit.Supplied[string]
details commandkit.Supplied[bool]
check commandkit.Supplied[bool]
values commandkit.FlagValues[I] values commandkit.FlagValues[I]
againstValues commandkit.FlagValues[I] againstValues commandkit.FlagValues[I]
plan *configOperationPlan[I, S] plan *configOperationPlan[I, S]
@@ -41,6 +43,8 @@ type configOperationPlan[I, S any] struct {
path configkit.Path path configkit.Path
pathSet bool pathSet bool
sourceID commandkit.Supplied[string] sourceID commandkit.Supplied[string]
details bool
check bool
list bool list bool
help bool help bool
historical bool historical bool
@@ -78,6 +82,8 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
} }
if operation == Diff { if operation == Diff {
flags = append(flags, flags = append(flags,
commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native before and after difference evidence."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
commandkit.BoolFlag(commandkit.FlagMeta{Name: "check", Usage: "Return status 3 when differences are found."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).check = value }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString, commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString,
func(input **configCommandInput[I, S], value commandkit.Supplied[string]) { func(input **configCommandInput[I, S], value commandkit.Supplied[string]) {
(*input).against.Config = value (*input).against.Config = value
@@ -248,6 +254,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
path: path, path: path,
pathSet: input.path.Set, pathSet: input.path.Set,
sourceID: input.sourceID, sourceID: input.sourceID,
details: input.details.Value,
check: input.check.Value,
list: input.list.Value, list: input.list.Value,
historical: input.historical.Value, historical: input.historical.Value,
under: under, under: under,