Verify the reliability sprint and guard release checkout validation

This commit is contained in:
2026-11-16 15:19:59 +00:00
parent 11f314d995
commit a355f05a27
15 changed files with 322 additions and 334 deletions

View File

@@ -16,7 +16,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
commandkit "go.maximumdirect.net/commandkit"
@@ -138,21 +137,63 @@ pipeline:
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
}
markdown := string(reporting.Markdown(result.Snapshot))
if !strings.Contains(markdown, `1: glossary (party/../\|&lt;tag&gt;)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
t.Fatalf("report labels: %s", markdown)
type coordinates struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id"`
}
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
if readErr != nil {
t.Fatal(readErr)
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
check := func(name string, entries []coordinates) {
t.Helper()
seen := make(map[int]bool)
for _, entry := range entries {
id, ok := expected[entry.StageIndex]
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
t.Fatalf("%s has wrong stage association: %+v", name, entry)
}
seen[entry.StageIndex] = true
}
var compact bytes.Buffer
if err := json.Compact(&compact, raw); err != nil {
t.Fatalf("decode %s: %v", artifact, err)
if len(seen) != len(expected) {
t.Fatalf("%s missing stage associations: %v", name, seen)
}
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw)
}
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
if err != nil {
t.Fatal(err)
}
var ledger []coordinates
if err := json.Unmarshal(raw, &ledger); err != nil {
t.Fatal(err)
}
check("ledger", ledger)
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
if err != nil {
t.Fatal(err)
}
var calls struct {
Calls []struct {
Context struct {
coordinates
Role string
}
}
}
if err := json.Unmarshal(raw, &calls); err != nil {
t.Fatal(err)
}
var locations []coordinates
roles := make(map[int]map[string]bool)
for _, call := range calls.Calls {
location := call.Context.coordinates
locations = append(locations, location)
if roles[location.StageIndex] == nil {
roles[location.StageIndex] = make(map[string]bool)
}
roles[location.StageIndex][call.Context.Role] = true
}
check("LLM observations", locations)
for index := range expected {
if !roles[index]["proposal"] || !roles[index]["validation"] {
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
}
}
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {

View File

@@ -6,6 +6,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
@@ -23,7 +24,7 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
return main
}
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
func TestResourceCommandsRejectInvalidOperatorPrompts(t *testing.T) {
definitions := map[string]string{
"malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
@@ -40,6 +41,28 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
}
}
func TestMalformedCompanionKeepsDelegatedExitStatusVisible(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles: [\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
if err := os.WriteFile(transcript, []byte("[]\n"), 0600); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
args []string
status int
}{
{[]string{"promptkit", "list", "prompts"}, 1},
{[]string{"config", "show", "--scope", "promptkit"}, 1},
{[]string{"process", transcript}, 2},
} {
var stdout, stderr bytes.Buffer
code := Run(context.Background(), append(tc.args, "--config", main), &stdout, &stderr)
if code != tc.status || !strings.Contains(stderr.String(), "yaml.invalid") || stdout.Len() != 0 {
t.Fatalf("%v: status=%d want=%d stdout=%q stderr=%q", tc.args, code, tc.status, stdout.String(), stderr.String())
}
}
}
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json")