Verify the reliability sprint and guard release checkout validation
This commit is contained in:
@@ -16,7 +16,6 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
@@ -138,21 +137,63 @@ pipeline:
|
||||
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
||||
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
||||
}
|
||||
markdown := string(reporting.Markdown(result.Snapshot))
|
||||
if !strings.Contains(markdown, `1: glossary (party/../\|<tag>)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
|
||||
t.Fatalf("report labels: %s", markdown)
|
||||
type coordinates struct {
|
||||
StageIndex int `json:"stage_index"`
|
||||
StageID *string `json:"stage_id"`
|
||||
}
|
||||
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
|
||||
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
|
||||
check := func(name string, entries []coordinates) {
|
||||
t.Helper()
|
||||
seen := make(map[int]bool)
|
||||
for _, entry := range entries {
|
||||
id, ok := expected[entry.StageIndex]
|
||||
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
|
||||
t.Fatalf("%s has wrong stage association: %+v", name, entry)
|
||||
}
|
||||
seen[entry.StageIndex] = true
|
||||
}
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, raw); err != nil {
|
||||
t.Fatalf("decode %s: %v", artifact, err)
|
||||
if len(seen) != len(expected) {
|
||||
t.Fatalf("%s missing stage associations: %v", name, seen)
|
||||
}
|
||||
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
|
||||
t.Fatalf("%s missing attribution: %s", artifact, raw)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ledger []coordinates
|
||||
if err := json.Unmarshal(raw, &ledger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check("ledger", ledger)
|
||||
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls struct {
|
||||
Calls []struct {
|
||||
Context struct {
|
||||
coordinates
|
||||
Role string
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal(raw, &calls); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var locations []coordinates
|
||||
roles := make(map[int]map[string]bool)
|
||||
for _, call := range calls.Calls {
|
||||
location := call.Context.coordinates
|
||||
locations = append(locations, location)
|
||||
if roles[location.StageIndex] == nil {
|
||||
roles[location.StageIndex] = make(map[string]bool)
|
||||
}
|
||||
roles[location.StageIndex][call.Context.Role] = true
|
||||
}
|
||||
check("LLM observations", locations)
|
||||
for index := range expected {
|
||||
if !roles[index]["proposal"] || !roles[index]["validation"] {
|
||||
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||
|
||||
Reference in New Issue
Block a user