Verify the reliability sprint and guard release checkout validation

This commit is contained in:
2026-11-16 15:19:59 +00:00
parent 11f314d995
commit a355f05a27
15 changed files with 322 additions and 334 deletions

View File

@@ -16,7 +16,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
commandkit "go.maximumdirect.net/commandkit"
@@ -138,21 +137,63 @@ pipeline:
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
}
markdown := string(reporting.Markdown(result.Snapshot))
if !strings.Contains(markdown, `1: glossary (party/../\|&lt;tag&gt;)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
t.Fatalf("report labels: %s", markdown)
type coordinates struct {
StageIndex int `json:"stage_index"`
StageID *string `json:"stage_id"`
}
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
if readErr != nil {
t.Fatal(readErr)
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
check := func(name string, entries []coordinates) {
t.Helper()
seen := make(map[int]bool)
for _, entry := range entries {
id, ok := expected[entry.StageIndex]
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
t.Fatalf("%s has wrong stage association: %+v", name, entry)
}
seen[entry.StageIndex] = true
}
var compact bytes.Buffer
if err := json.Compact(&compact, raw); err != nil {
t.Fatalf("decode %s: %v", artifact, err)
if len(seen) != len(expected) {
t.Fatalf("%s missing stage associations: %v", name, seen)
}
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw)
}
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
if err != nil {
t.Fatal(err)
}
var ledger []coordinates
if err := json.Unmarshal(raw, &ledger); err != nil {
t.Fatal(err)
}
check("ledger", ledger)
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
if err != nil {
t.Fatal(err)
}
var calls struct {
Calls []struct {
Context struct {
coordinates
Role string
}
}
}
if err := json.Unmarshal(raw, &calls); err != nil {
t.Fatal(err)
}
var locations []coordinates
roles := make(map[int]map[string]bool)
for _, call := range calls.Calls {
location := call.Context.coordinates
locations = append(locations, location)
if roles[location.StageIndex] == nil {
roles[location.StageIndex] = make(map[string]bool)
}
roles[location.StageIndex][call.Context.Role] = true
}
check("LLM observations", locations)
for index := range expected {
if !roles[index]["proposal"] || !roles[index]["validation"] {
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
}
}
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {

View File

@@ -6,6 +6,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
@@ -23,7 +24,7 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
return main
}
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
func TestResourceCommandsRejectInvalidOperatorPrompts(t *testing.T) {
definitions := map[string]string{
"malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
@@ -40,6 +41,28 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
}
}
func TestMalformedCompanionKeepsDelegatedExitStatusVisible(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles: [\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
if err := os.WriteFile(transcript, []byte("[]\n"), 0600); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
args []string
status int
}{
{[]string{"promptkit", "list", "prompts"}, 1},
{[]string{"config", "show", "--scope", "promptkit"}, 1},
{[]string{"process", transcript}, 2},
} {
var stdout, stderr bytes.Buffer
code := Run(context.Background(), append(tc.args, "--config", main), &stdout, &stderr)
if code != tc.status || !strings.Contains(stderr.String(), "yaml.invalid") || stdout.Len() != 0 {
t.Fatalf("%v: status=%d want=%d stdout=%q stderr=%q", tc.args, code, tc.status, stdout.String(), stderr.String())
}
}
}
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json")

View File

@@ -234,7 +234,11 @@ func recordLedgerDecision(decision runner.ValidatorDecisionRecord, deterministic
ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName,
Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message,
}
if decision.ExecutionClass == validatormetadata.ExecutionClassLLMBacked {
class := decision.ExecutionClass
if class == "" {
class = validatormetadata.ClassForKey(decision.ValidatorName)
}
if class == validatormetadata.ExecutionClassLLMBacked {
*llm = append(*llm, record)
} else {
*deterministic = append(*deterministic, record)

View File

@@ -68,17 +68,6 @@ func TestLookupUnknownSchema(t *testing.T) {
}
}
func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
for _, schema := range Registered() {
if len(schema.JSONSchema) == 0 {
t.Fatalf("expected non-empty JSON schema for %q", schema.ID)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("expected valid JSON schema for %q", schema.ID)
}
}
}
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
schema, ok := Lookup(key)

View File

@@ -7,7 +7,6 @@ import (
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"
"testing"
@@ -451,34 +450,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
}
}
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
got := 0
for got < want {
select {
case <-entered:
got++
case <-deadline:
t.Fatalf("timed out waiting for %d entries (%s), got %d", want, label, got)
}
}
}
func waitForRunnerSectionIDs(t *testing.T, started <-chan int, want map[int]struct{}) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
seen := map[int]struct{}{}
for len(seen) < len(want) {
select {
case id := <-started:
seen[id] = struct{}{}
case <-deadline:
t.Fatalf("timed out waiting for section IDs %v, got %v", mapKeys(want), mapKeys(seen))
}
}
}
func mapKeys(values map[int]struct{}) []int {
keys := make([]int, 0, len(values))
for key := range values {
@@ -563,60 +534,6 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
}
}
type stageAwareStructuredClient struct {
startedSection chan int
releaseValidation <-chan struct{}
eventSink chan<- string
}
type captureContractStructuredClient struct {
lastRequest contracts.StructuredCompletionRequest
}
func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.lastRequest = req
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
}
return contracts.StructuredCompletionResponse{}, nil
}
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
section := 0
if req.Context.SectionIndex != nil {
section = *req.Context.SectionIndex
}
if c.startedSection != nil {
select {
case c.startedSection <- section:
default:
}
}
if c.eventSink != nil {
c.eventSink <- "v" + strconv.Itoa(section)
}
if c.releaseValidation != nil {
select {
case <-c.releaseValidation:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunnerSkippedRecorded(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{

View File

@@ -76,19 +76,27 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
}
}
func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) {
all := RegisteredMetadata()
if len(all) != 11 {
t.Fatalf("prompt count: %d", len(all))
func TestRegistryMetadataIdentifiesRequiredPrompts(t *testing.T) {
required := []string{
"audita/modules.glossary.proposal", "audita/modules.glossary.repair",
"audita/modules.grammar.proposal", "audita/modules.grammar.repair",
"audita/modules.homophones.proposal", "audita/modules.homophones.repair",
"audita/modules.spoken_word.proposal", "audita/modules.spoken_word.repair",
"audita/validators.editorial_review", "audita/validators.meaning_reversal_review",
"audita/validators.spoken_form_plausibility",
}
for _, meta := range all {
if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" {
t.Fatalf("metadata: %+v", meta)
}
if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil {
t.Fatal(err)
for _, id := range required {
metadata, ok := LookupMetadata(id)
if !ok || metadata.PromptID != id || metadata.PromptSource != SourceBuiltin ||
metadata.EmbeddedPath != AssetRoot+"/"+strings.TrimPrefix(id, "audita/")+".yaml" {
t.Fatalf("required prompt metadata for %q: %+v", id, metadata)
}
}
all := RegisteredMetadata()
all[0].PromptID = "changed"
if again := RegisteredMetadata(); again[0].PromptID == "changed" {
t.Fatal("metadata enumeration shares mutable state")
}
}
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {

View File

@@ -1,57 +0,0 @@
package testsupport
import (
"os"
"path/filepath"
"strings"
"testing"
)
func ReadFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read file %q: %v", path, err)
}
return data
}
func OnlyRunDir(t *testing.T, workDir string) string {
t.Helper()
entries, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work dir %q: %v", workDir, err)
}
dirs := make([]string, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, filepath.Join(workDir, e.Name()))
}
}
if len(dirs) != 1 {
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
}
return dirs[0]
}
func AssertNoSecretInFile(t *testing.T, path, secret string) {
t.Helper()
raw := string(ReadFile(t, path))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in %s", path)
}
}
func AssertNoSecretInTree(t *testing.T, root, secret string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw, readErr := os.ReadFile(path)
if readErr == nil && strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in %s", path)
}
return nil
})
}

View File

@@ -9,104 +9,33 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/non_empty_resulting_segment"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/spoken_form_plausibility"
)
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range []string{
KeyProposalShape,
KeyOriginalTextPresence,
KeyNonEmptyResultingSegment,
KeyNoEffect,
KeyProtectedTerms,
KeySpokenFormPlausibility,
KeyMeaningReversalReview,
KeyEditorialReview,
func TestRegisteredValidatorsUseDocumentedRoles(t *testing.T) {
registry := NewBuiltInRegistry()
for _, tc := range []struct {
key string
role validatormetadata.ExecutionClass
}{
{KeyProposalShape, validatormetadata.ExecutionClassDeterministic},
{KeyOriginalTextPresence, validatormetadata.ExecutionClassDeterministic},
{KeyNonEmptyResultingSegment, validatormetadata.ExecutionClassDeterministic},
{KeyNoEffect, validatormetadata.ExecutionClassDeterministic},
{KeyProtectedTerms, validatormetadata.ExecutionClassDeterministic},
{KeySpokenFormPlausibility, validatormetadata.ExecutionClassLLMBacked},
{KeyMeaningReversalReview, validatormetadata.ExecutionClassLLMBacked},
{KeyEditorialReview, validatormetadata.ExecutionClassLLMBacked},
} {
def, ok := r.Lookup(key)
if !ok {
t.Fatalf("expected validator key %q to be registered", key)
}
v, err := def.Build()
if err != nil {
t.Fatalf("build validator %q: %v", key, err)
}
if v.Name() != key {
t.Fatalf("expected validator name %q, got %q", key, v.Name())
}
}
}
func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
type validatorCtor struct {
name string
key string
build func() (frameworkvalidators.Validator, error)
wantClass validatormetadata.ExecutionClass
}
cases := []validatorCtor{
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "non_empty_resulting_segment", key: KeyNonEmptyResultingSegment, build: non_empty_resulting_segment.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "no_effect", key: KeyNoEffect, build: no_effect.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "protected_terms", key: KeyProtectedTerms, build: protected_terms.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "spoken_form_plausibility", key: KeySpokenFormPlausibility, build: spoken_form_plausibility.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
{name: "meaning_reversal_review", key: KeyMeaningReversalReview, build: meaning_reversal_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
{name: "editorial_review", key: KeyEditorialReview, build: editorial_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
}
for _, tc := range cases {
v, err := tc.build()
if err != nil {
t.Fatalf("%s: build: %v", tc.name, err)
}
if v.Name() != tc.key {
t.Fatalf("%s: expected key %q, got %q", tc.name, tc.key, v.Name())
}
if got := validatormetadata.ClassOf(v); got != tc.wantClass {
t.Fatalf("%s: expected class %q, got %q", tc.name, tc.wantClass, got)
}
}
}
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
t.Fatalf("must build %q: %v", key, err)
}
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
}
want := validatormetadata.ClassForKey(key)
if got := validatormetadata.ClassOf(v); got != want {
t.Fatalf("expected class %q for %q, got %q", want, key, got)
}
}
}
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
t.Fatalf("must build %q: %v", key, err)
}
fromKey := validatormetadata.ClassForKey(key)
fromInstance := validatormetadata.ClassOf(v)
if fromInstance != fromKey {
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
}
t.Run(tc.key, func(t *testing.T) {
v, err := registry.MustBuild(tc.key)
if err != nil {
t.Fatal(err)
}
if v.Name() != tc.key || validatormetadata.ClassOf(v) != tc.role {
t.Fatalf("registered name=%q role=%q, want %q and %q", v.Name(), validatormetadata.ClassOf(v), tc.key, tc.role)
}
})
}
}