Verify the reliability sprint and guard release checkout validation

This commit is contained in:
2026-11-16 15:19:59 +00:00
parent 11f314d995
commit a355f05a27
15 changed files with 322 additions and 334 deletions

View File

@@ -24,7 +24,8 @@ steps:
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}" test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
git checkout --detach "$${tag_commit}" git checkout --detach "$${tag_commit}"
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}" test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
test -z "$(git status --porcelain --untracked-files=all)" checkout_status=$(git status --porcelain --untracked-files=all) || exit $?
test -z "$${checkout_status}"
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX") bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
trap 'rm -rf "$${bootstrap_dir}"' EXIT trap 'rm -rf "$${bootstrap_dir}"' EXIT

View File

@@ -2,6 +2,7 @@ package main
import ( import (
"context" "context"
"io"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
@@ -10,11 +11,11 @@ import (
) )
func main() { func main() {
os.Exit(run()) os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
} }
func run() int { func run(args []string, stdout, stderr io.Writer) int {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop() defer stop()
return cli.Run(ctx, os.Args[1:], os.Stdout, os.Stderr) return cli.Run(ctx, args, stdout, stderr)
} }

View File

@@ -12,7 +12,6 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"os/exec" "os/exec"
"os/signal"
"path/filepath" "path/filepath"
"reflect" "reflect"
"strconv" "strconv"
@@ -41,10 +40,7 @@ func TestProcessSubprocessHelper(t *testing.T) {
} }
syscall.Umask(int(mask)) syscall.Umask(int(mask))
} }
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) os.Exit(run(os.Args[i+1:], os.Stdout, os.Stderr))
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
stop()
os.Exit(code)
} }
} }
os.Exit(2) os.Exit(2)
@@ -237,7 +233,6 @@ type processFixture struct {
dir string dir string
config string config string
transcript string transcript string
glossary string
output string output string
provider *fixtureProvider provider *fixtureProvider
server *httptest.Server server *httptest.Server
@@ -282,11 +277,8 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil { if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
t.Fatal(err) t.Fatal(err)
} }
glossary := filepath.Join(dir, "glossary.yaml")
if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil { return processFixture{dir: dir, config: configPath, transcript: transcript,
t.Fatal(err)
}
return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary,
output: filepath.Join(dir, "output.json"), provider: p, server: server} output: filepath.Join(dir, "output.json"), provider: p, server: server}
} }
@@ -314,25 +306,6 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (
return code, stdout.String(), stderr.String() return code, stdout.String(), stderr.String()
} }
func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) {
t.Helper()
command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view)
command.Dir = f.dir
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stdout, stderr bytes.Buffer
command.Stdout, command.Stderr = &stdout, &stderr
err := command.Run()
code := 0
if err != nil {
if exit, ok := err.(*exec.ExitError); ok {
code = exit.ExitCode()
} else {
t.Fatal(err)
}
}
return code, stdout.String(), stderr.String()
}
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument { func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
t.Helper() t.Helper()
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml")) raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
@@ -879,7 +852,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
for _, selection := range []struct{ reporting, debug bool }{ for _, selection := range []struct{ reporting, debug bool }{
{false, false}, {false, true}, {true, false}, {true, true}, {false, false}, {false, true}, {true, false}, {true, true},
} { } {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") f := newProcessFixture(t, "normal", "[grammar]")
reportDir := filepath.Join(f.dir, "missing") reportDir := filepath.Join(f.dir, "missing")
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir} extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
@@ -894,7 +867,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
extra = append(extra, "--debug") extra = append(extra, "--debug")
} }
code, stdout, stderr := f.run(t, true, extra...) code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
} }
output, err := os.ReadFile(f.output) output, err := os.ReadFile(f.output)
@@ -1072,8 +1045,9 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
t.Fatal(err) t.Fatal(err)
} }
err := command.Wait() err := command.Wait()
if ctx.Err() != nil || err == nil { var exit *exec.ExitError
t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String()) if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 1 {
t.Fatalf("interrupt did not reach the failure exit: %v %v stderr=%q", err, ctx.Err(), stderr.String())
} }
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 { if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
t.Fatalf("cancellation report: %+v", document) t.Fatalf("cancellation report: %+v", document)
@@ -1556,11 +1530,13 @@ func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) {
case <-ctx.Done(): case <-ctx.Done():
t.Fatal("proposal progress did not reach stderr while the provider was blocked") t.Fatal("proposal progress did not reach stderr while the provider was blocked")
} }
if err := cmd.Process.Signal(os.Interrupt); err != nil { if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := cmd.Wait(); err == nil || ctx.Err() != nil { err = cmd.Wait()
t.Fatalf("interrupt did not complete the invocation: %v", err) var exit *exec.ExitError
if !errors.As(err, &exit) || exit.ExitCode() != 1 || ctx.Err() != nil {
t.Fatalf("termination did not reach the failure exit: %v", err)
} }
} }

View File

@@ -16,7 +16,8 @@ repair. An expired parent context stops processing and retains any native cause.
With an active parent, PromptKit's operation deadline remains recoverable even With an active parent, PromptKit's operation deadline remains recoverable even
when a preparation error wraps it. A native transport timeout affects the HTTP when a preparation error wraps it. A native transport timeout affects the HTTP
attempt and is recorded as a transport failure. The prepared notification attempt and is recorded as a transport failure. The prepared notification
provides the actual request snapshot; failures before notification have no snapshot. supplies prompt metadata to observers. With debug capture enabled, it also
supplies the actual request messages when preparation reaches that point.
PromptKit owns backend and credential selection. When a required credential is absent, PromptKit owns backend and credential selection. When a required credential is absent,
the CLI identifies the role, profile, and environment variable using the native the CLI identifies the role, profile, and environment variable using the native
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md) diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)

View File

@@ -1,5 +1,8 @@
# Migrating to Audita v2 # Migrating to Audita v2
This guide describes the historical v2 release. For current commands,
configuration, and example projects, use the [v3 migration guide](v3.md).
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution. Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
Version-1 configuration files, environment overlays, and retired flags are not Version-1 configuration files, environment overlays, and retired flags are not
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or

View File

@@ -52,7 +52,7 @@ record the mismatch explicitly. A test that asserts only nonzero status does
not verify this fix. not verify this fix.
**Complete.** The upstream handoff is recorded below. The exit-status fix **Complete.** The upstream handoff is recorded below. The exit-status fix
remains outstanding for the final verification pass. remains outstanding after the final verification pass.
Build a temporary executable with `go build -o /tmp/audita-inspection ./cmd/audita`. Build a temporary executable with `go build -o /tmp/audita-inspection ./cmd/audita`.
In a scratch directory, put `version: 3` in `config.yml`, `profiles: [` in the In a scratch directory, put `version: 3` in `config.yml`, `profiles: [` in the
@@ -747,3 +747,68 @@ Audit #28, #29, #58, #60–62 and sprint closure.
The completed sprint should leave the target architecture reachable through The completed sprint should leave the target architecture reachable through
`audita process` and the standard delegated commands. Report checks and `audita process` and the standard delegated commands. Report checks and
measurements actually performed, including any outstanding external dependency. measurements actually performed, including any outstanding external dependency.
**Complete.** The integrated Audita work and release bootstrap passed checks.
The delegated inspection exit status remains an outstanding upstream item.
The bootstrap rejects a dirty checkout and a failed status command with no
output. A clean fixture reaches ReleaseKit with the prepared checkout and
expected commit arguments. Process fixtures share the production entry point
and check interrupt and termination exit status. Registry coverage checks
effective roles, required prompt identities and preparation, schema identity,
and independent copies. The ordered processing fixture decodes debug artifact
coordinates. The full repair/filter/later-failure case remains separate from
the smaller output-choice matrix. Unused fixture helpers and the legacy
glossary input were removed.
### Delegated inspection status
The [CommandKit upstream tags](https://forge.maximumdirect.net/base-layer/commandkit/tags)
confirmed `v0.7.0` is the latest release of both `ckprompt` and `ckconfig`.
The built Audita executable reproduced the Stage 1 mismatch:
| Invalid input | Path | Observed status and diagnostic |
| --- | --- | --- |
| Malformed automatic `prompts.yml` | `process` | 2, `yaml.invalid` |
| Malformed automatic `prompts.yml` | `promptkit list prompts` | 1, `yaml.invalid` |
| Malformed automatic `prompts.yml` | `config show/sources/validate/explain/diff --scope promptkit` | 1, `yaml.invalid` |
| Malformed explicit app main file | `config show/sources/validate/explain/diff --scope app` | 1, `yaml.invalid` |
Caller cancellation remains status 1 in the shared CLI and subprocess tests.
The CommandKit integrations still own the fix to return status 2 for typed
operator-input failures. Audita retains its native delegations. The expected
upstream change and reproduction remain in the Stage 1 handoff.
### Catalog closure
The table identifies the implementing stages and exceptions without changing
the historical audit observations. Each stage records its verification above.
| Catalog findings | Implementing stages or disposition |
| --- | --- |
| #1, #6–8, #16, #26, #42 | Stage 4. Only cooperative cancellation boundaries from #6 were selected. |
| #2, #4 | No implementation. The existing catalog dispositions stand. |
| #3, #9, #63–65 | Stages 2 and 14. |
| #5, #57 | Stage 19. |
| #10–11, #27, #40, #49 | Stages 9, 11, and 17. |
| #12, #18–19, #38, #47 | Stage 8. Only punctuation-edge handling from #18 was selected. |
| #13, #31, #51–52, #54 | Stages 2 and 5. |
| #14–15, #22, #30–32, #39, #41 | Stages 5, 12, and 15–18. |
| #17, #21, #35, #53, #59, #62 | Stages 5 and 6, with final caller checks in Stage 20. |
| #20, #24 | Stages 7 and 11 for processing; Stage 1 and the upstream disposition above for delegated inspection. |
| #23, #25, #44 | Stages 12, 18, and 19. |
| #28–29, #58, #60–61 | Stage 20. |
| #33, #43 | Stages 3 and 18. |
| #34, #45, #48 | Stage 14. |
| #36 | Stage 10. |
| #37, #50, #55–56 | Stage 11. |
| #46 | Stage 13. |
Verified with Go 1.27.3 by building the CLI, running `go test -count=1 ./...`
offline, and running targeted race checks for the runner, LLM adapter,
validators, reports, diagnostics, glossary, and CLI. The final review
checked Go formatting, whitespace, and 278 relative documentation links
and exercised the example configurations. Measurements above cover merge
and glossary allocation reductions, validation context, repair request and
artifact costs, bounded section contention, and retained evidence by output
mode. These are synthetic measurements. No release was published.

View File

@@ -16,7 +16,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics" "gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
commandkit "go.maximumdirect.net/commandkit" commandkit "go.maximumdirect.net/commandkit"
@@ -138,21 +137,63 @@ pipeline:
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" { if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries) t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
} }
markdown := string(reporting.Markdown(result.Snapshot)) type coordinates struct {
if !strings.Contains(markdown, `1: glossary (party/../\|&lt;tag&gt;)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") { StageIndex int `json:"stage_index"`
t.Fatalf("report labels: %s", markdown) StageID *string `json:"stage_id"`
} }
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} { expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact)) check := func(name string, entries []coordinates) {
if readErr != nil { t.Helper()
t.Fatal(readErr) seen := make(map[int]bool)
for _, entry := range entries {
id, ok := expected[entry.StageIndex]
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
t.Fatalf("%s has wrong stage association: %+v", name, entry)
}
seen[entry.StageIndex] = true
} }
var compact bytes.Buffer if len(seen) != len(expected) {
if err := json.Compact(&compact, raw); err != nil { t.Fatalf("%s missing stage associations: %v", name, seen)
t.Fatalf("decode %s: %v", artifact, err)
} }
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) { }
t.Fatalf("%s missing attribution: %s", artifact, raw) raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
if err != nil {
t.Fatal(err)
}
var ledger []coordinates
if err := json.Unmarshal(raw, &ledger); err != nil {
t.Fatal(err)
}
check("ledger", ledger)
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
if err != nil {
t.Fatal(err)
}
var calls struct {
Calls []struct {
Context struct {
coordinates
Role string
}
}
}
if err := json.Unmarshal(raw, &calls); err != nil {
t.Fatal(err)
}
var locations []coordinates
roles := make(map[int]map[string]bool)
for _, call := range calls.Calls {
location := call.Context.coordinates
locations = append(locations, location)
if roles[location.StageIndex] == nil {
roles[location.StageIndex] = make(map[string]bool)
}
roles[location.StageIndex][call.Context.Role] = true
}
check("LLM observations", locations)
for index := range expected {
if !roles[index]["proposal"] || !roles[index]["validation"] {
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
} }
} }
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) { if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {

View File

@@ -6,6 +6,7 @@ import (
"errors" "errors"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
commandkit "go.maximumdirect.net/commandkit" commandkit "go.maximumdirect.net/commandkit"
@@ -23,7 +24,7 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
return main return main
} }
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) { func TestResourceCommandsRejectInvalidOperatorPrompts(t *testing.T) {
definitions := map[string]string{ definitions := map[string]string{
"malformed operator YAML": "profiles: [\n", "malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n", "invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
@@ -40,6 +41,28 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
} }
} }
func TestMalformedCompanionKeepsDelegatedExitStatusVisible(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles: [\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
if err := os.WriteFile(transcript, []byte("[]\n"), 0600); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
args []string
status int
}{
{[]string{"promptkit", "list", "prompts"}, 1},
{[]string{"config", "show", "--scope", "promptkit"}, 1},
{[]string{"process", transcript}, 2},
} {
var stdout, stderr bytes.Buffer
code := Run(context.Background(), append(tc.args, "--config", main), &stdout, &stderr)
if code != tc.status || !strings.Contains(stderr.String(), "yaml.invalid") || stdout.Len() != 0 {
t.Fatalf("%v: status=%d want=%d stdout=%q stderr=%q", tc.args, code, tc.status, stdout.String(), stderr.String())
}
}
}
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) { func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n") main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
transcript := filepath.Join(filepath.Dir(main), "transcript.json") transcript := filepath.Join(filepath.Dir(main), "transcript.json")

View File

@@ -234,7 +234,11 @@ func recordLedgerDecision(decision runner.ValidatorDecisionRecord, deterministic
ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName, ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName,
Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message, Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message,
} }
if decision.ExecutionClass == validatormetadata.ExecutionClassLLMBacked { class := decision.ExecutionClass
if class == "" {
class = validatormetadata.ClassForKey(decision.ValidatorName)
}
if class == validatormetadata.ExecutionClassLLMBacked {
*llm = append(*llm, record) *llm = append(*llm, record)
} else { } else {
*deterministic = append(*deterministic, record) *deterministic = append(*deterministic, record)

View File

@@ -68,17 +68,6 @@ func TestLookupUnknownSchema(t *testing.T) {
} }
} }
func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
for _, schema := range Registered() {
if len(schema.JSONSchema) == 0 {
t.Fatalf("expected non-empty JSON schema for %q", schema.ID)
}
if !json.Valid(schema.JSONSchema) {
t.Fatalf("expected valid JSON schema for %q", schema.ID)
}
}
}
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) { func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} { for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
schema, ok := Lookup(key) schema, ok := Lookup(key)

View File

@@ -7,7 +7,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"reflect" "reflect"
"strconv"
"strings" "strings"
"sync" "sync"
"testing" "testing"
@@ -451,34 +450,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
} }
} }
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
got := 0
for got < want {
select {
case <-entered:
got++
case <-deadline:
t.Fatalf("timed out waiting for %d entries (%s), got %d", want, label, got)
}
}
}
func waitForRunnerSectionIDs(t *testing.T, started <-chan int, want map[int]struct{}) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
seen := map[int]struct{}{}
for len(seen) < len(want) {
select {
case id := <-started:
seen[id] = struct{}{}
case <-deadline:
t.Fatalf("timed out waiting for section IDs %v, got %v", mapKeys(want), mapKeys(seen))
}
}
}
func mapKeys(values map[int]struct{}) []int { func mapKeys(values map[int]struct{}) []int {
keys := make([]int, 0, len(values)) keys := make([]int, 0, len(values))
for key := range values { for key := range values {
@@ -563,60 +534,6 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
} }
} }
type stageAwareStructuredClient struct {
startedSection chan int
releaseValidation <-chan struct{}
eventSink chan<- string
}
type captureContractStructuredClient struct {
lastRequest contracts.StructuredCompletionRequest
}
func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.lastRequest = req
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
}
return contracts.StructuredCompletionResponse{}, nil
}
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
section := 0
if req.Context.SectionIndex != nil {
section = *req.Context.SectionIndex
}
if c.startedSection != nil {
select {
case c.startedSection <- section:
default:
}
}
if c.eventSink != nil {
c.eventSink <- "v" + strconv.Itoa(section)
}
if c.releaseValidation != nil {
select {
case <-c.releaseValidation:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunnerSkippedRecorded(t *testing.T) { func TestRunnerSkippedRecorded(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}} transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{

View File

@@ -76,19 +76,27 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
} }
} }
func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) { func TestRegistryMetadataIdentifiesRequiredPrompts(t *testing.T) {
all := RegisteredMetadata() required := []string{
if len(all) != 11 { "audita/modules.glossary.proposal", "audita/modules.glossary.repair",
t.Fatalf("prompt count: %d", len(all)) "audita/modules.grammar.proposal", "audita/modules.grammar.repair",
"audita/modules.homophones.proposal", "audita/modules.homophones.repair",
"audita/modules.spoken_word.proposal", "audita/modules.spoken_word.repair",
"audita/validators.editorial_review", "audita/validators.meaning_reversal_review",
"audita/validators.spoken_form_plausibility",
} }
for _, meta := range all { for _, id := range required {
if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" { metadata, ok := LookupMetadata(id)
t.Fatalf("metadata: %+v", meta) if !ok || metadata.PromptID != id || metadata.PromptSource != SourceBuiltin ||
} metadata.EmbeddedPath != AssetRoot+"/"+strings.TrimPrefix(id, "audita/")+".yaml" {
if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil { t.Fatalf("required prompt metadata for %q: %+v", id, metadata)
t.Fatal(err)
} }
} }
all := RegisteredMetadata()
all[0].PromptID = "changed"
if again := RegisteredMetadata(); again[0].PromptID == "changed" {
t.Fatal("metadata enumeration shares mutable state")
}
} }
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) { func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {

View File

@@ -1,57 +0,0 @@
package testsupport
import (
"os"
"path/filepath"
"strings"
"testing"
)
func ReadFile(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("failed to read file %q: %v", path, err)
}
return data
}
func OnlyRunDir(t *testing.T, workDir string) string {
t.Helper()
entries, err := os.ReadDir(workDir)
if err != nil {
t.Fatalf("failed to read work dir %q: %v", workDir, err)
}
dirs := make([]string, 0, len(entries))
for _, e := range entries {
if e.IsDir() {
dirs = append(dirs, filepath.Join(workDir, e.Name()))
}
}
if len(dirs) != 1 {
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
}
return dirs[0]
}
func AssertNoSecretInFile(t *testing.T, path, secret string) {
t.Helper()
raw := string(ReadFile(t, path))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in %s", path)
}
}
func AssertNoSecretInTree(t *testing.T, root, secret string) {
t.Helper()
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
if err != nil || d == nil || d.IsDir() {
return nil
}
raw, readErr := os.ReadFile(path)
if readErr == nil && strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in %s", path)
}
return nil
})
}

View File

@@ -9,104 +9,33 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/no_effect"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/non_empty_resulting_segment"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/original_text_presence"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/proposal_shape"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/spoken_form_plausibility"
) )
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) { func TestRegisteredValidatorsUseDocumentedRoles(t *testing.T) {
r := NewBuiltInRegistry() registry := NewBuiltInRegistry()
for _, key := range []string{ for _, tc := range []struct {
KeyProposalShape, key string
KeyOriginalTextPresence, role validatormetadata.ExecutionClass
KeyNonEmptyResultingSegment, }{
KeyNoEffect, {KeyProposalShape, validatormetadata.ExecutionClassDeterministic},
KeyProtectedTerms, {KeyOriginalTextPresence, validatormetadata.ExecutionClassDeterministic},
KeySpokenFormPlausibility, {KeyNonEmptyResultingSegment, validatormetadata.ExecutionClassDeterministic},
KeyMeaningReversalReview, {KeyNoEffect, validatormetadata.ExecutionClassDeterministic},
KeyEditorialReview, {KeyProtectedTerms, validatormetadata.ExecutionClassDeterministic},
{KeySpokenFormPlausibility, validatormetadata.ExecutionClassLLMBacked},
{KeyMeaningReversalReview, validatormetadata.ExecutionClassLLMBacked},
{KeyEditorialReview, validatormetadata.ExecutionClassLLMBacked},
} { } {
def, ok := r.Lookup(key) t.Run(tc.key, func(t *testing.T) {
if !ok { v, err := registry.MustBuild(tc.key)
t.Fatalf("expected validator key %q to be registered", key) if err != nil {
} t.Fatal(err)
v, err := def.Build() }
if err != nil { if v.Name() != tc.key || validatormetadata.ClassOf(v) != tc.role {
t.Fatalf("build validator %q: %v", key, err) t.Fatalf("registered name=%q role=%q, want %q and %q", v.Name(), validatormetadata.ClassOf(v), tc.key, tc.role)
} }
if v.Name() != key { })
t.Fatalf("expected validator name %q, got %q", key, v.Name())
}
}
}
func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
type validatorCtor struct {
name string
key string
build func() (frameworkvalidators.Validator, error)
wantClass validatormetadata.ExecutionClass
}
cases := []validatorCtor{
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "non_empty_resulting_segment", key: KeyNonEmptyResultingSegment, build: non_empty_resulting_segment.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "no_effect", key: KeyNoEffect, build: no_effect.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "protected_terms", key: KeyProtectedTerms, build: protected_terms.New, wantClass: validatormetadata.ExecutionClassDeterministic},
{name: "spoken_form_plausibility", key: KeySpokenFormPlausibility, build: spoken_form_plausibility.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
{name: "meaning_reversal_review", key: KeyMeaningReversalReview, build: meaning_reversal_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
{name: "editorial_review", key: KeyEditorialReview, build: editorial_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
}
for _, tc := range cases {
v, err := tc.build()
if err != nil {
t.Fatalf("%s: build: %v", tc.name, err)
}
if v.Name() != tc.key {
t.Fatalf("%s: expected key %q, got %q", tc.name, tc.key, v.Name())
}
if got := validatormetadata.ClassOf(v); got != tc.wantClass {
t.Fatalf("%s: expected class %q, got %q", tc.name, tc.wantClass, got)
}
}
}
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
t.Fatalf("must build %q: %v", key, err)
}
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
}
want := validatormetadata.ClassForKey(key)
if got := validatormetadata.ClassOf(v); got != want {
t.Fatalf("expected class %q for %q, got %q", want, key, got)
}
}
}
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
r := NewBuiltInRegistry()
for _, key := range r.RegisteredKeys() {
v, err := r.MustBuild(key)
if err != nil {
t.Fatalf("must build %q: %v", key, err)
}
fromKey := validatormetadata.ClassForKey(key)
fromInstance := validatormetadata.ClassOf(v)
if fromInstance != fromKey {
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
}
} }
} }

View File

@@ -4,7 +4,12 @@ import (
"encoding/json" "encoding/json"
"maps" "maps"
"os" "os"
"os/exec"
"path/filepath"
"strings"
"testing" "testing"
"gopkg.in/yaml.v3"
) )
func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) { func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) {
@@ -60,27 +65,6 @@ func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) {
} }
} }
func TestReleaseKitTargetMembership(t *testing.T) {
want := []string{"linux/amd64", "linux/arm64"}
for _, test := range []struct {
name string
targets []string
match bool
}{
{"missing amd64", []string{"linux/arm64", "linux/arm64"}, false},
{"missing arm64", []string{"linux/amd64", "linux/amd64"}, false},
{"additional target", []string{"linux/amd64", "linux/arm64", "darwin/arm64"}, false},
{"reordered", []string{"linux/arm64", "linux/amd64"}, true},
{"repeated", []string{"linux/amd64", "linux/arm64", "linux/amd64"}, true},
} {
t.Run(test.name, func(t *testing.T) {
if got := sameMembers(test.targets, want); got != test.match {
t.Fatalf("target membership for %q = %t, want %t", test.targets, got, test.match)
}
})
}
}
func sameMembers(got, want []string) bool { func sameMembers(got, want []string) bool {
gotSet := make(map[string]struct{}, len(got)) gotSet := make(map[string]struct{}, len(got))
for _, value := range got { for _, value := range got {
@@ -92,3 +76,106 @@ func sameMembers(got, want []string) bool {
} }
return maps.Equal(gotSet, wantSet) return maps.Equal(gotSet, wantSet)
} }
func TestReleaseBootstrapRequiresSuccessfulCleanStatus(t *testing.T) {
data, err := os.ReadFile("../.woodpecker/release.yml")
if err != nil {
t.Fatal(err)
}
var workflow struct {
Steps []struct {
Name string
Commands []string
}
}
if err := yaml.Unmarshal(data, &workflow); err != nil {
t.Fatal(err)
}
var script string
for _, step := range workflow.Steps {
if step.Name == "deliver" {
// Woodpecker removes one dollar from escaped shell variables.
script = strings.ReplaceAll(strings.Join(step.Commands, "\n"), "$${", "${")
}
}
if script == "" {
t.Fatal("delivery command is missing")
}
for _, tc := range []struct {
name string
status string
exit int
}{
{"clean", "clean", 0},
{"dirty", "dirty", 1},
{"status failed with no output", "failed", 93},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
workspace := filepath.Join(dir, "checkout")
if err := os.Mkdir(workspace, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(workspace, "go.mod"), []byte("module example.invalid/release\n"), 0600); err != nil {
t.Fatal(err)
}
writeCommand := func(name, script string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nset -eu\n"+script), 0700); err != nil {
t.Fatal(err)
}
}
writeCommand("git", `
case "$1" in
check-ref-format|fetch|checkout) ;;
rev-parse) printf '%s\n' "$CI_COMMIT_SHA" ;;
status)
test "$*" = 'status --porcelain --untracked-files=all'
case "$STATUS_FIXTURE" in
dirty) printf '%s\n' '?? untracked' ;;
failed) exit 93 ;;
esac
;;
*) exit 99 ;;
esac
`)
writeCommand("go", `
case "$*" in
'env GOHOSTOS') printf '%s\n' linux ;;
'env GOHOSTARCH') printf '%s\n' amd64 ;;
esac
printf '%s\n' "$*" >> "$GO_CALLS_FIXTURE"
`)
calls := filepath.Join(dir, "go-calls")
commit := strings.Repeat("a", 40)
command := exec.Command("sh", "-c", script)
command.Dir = workspace
command.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"),
"CI_WORKSPACE="+workspace, "CI_COMMIT_TAG=v1.2.3", "CI_COMMIT_SHA="+commit,
"STATUS_FIXTURE="+tc.status, "GO_CALLS_FIXTURE="+calls)
output, err := command.CombinedOutput()
exit := 0
if err != nil {
failure, ok := err.(*exec.ExitError)
if !ok {
t.Fatal(err)
}
exit = failure.ExitCode()
}
if exit != tc.exit {
t.Fatalf("exit=%d want %d: %s", exit, tc.exit, output)
}
transcript, err := os.ReadFile(calls)
if tc.exit != 0 {
if !os.IsNotExist(err) {
t.Fatalf("bootstrap reached Go after failed status: %s (%v)", transcript, err)
}
return
}
if err != nil || !strings.Contains(string(transcript), "ci-deliver --repo "+workspace+" --prepared --commit "+commit+" v1.2.3") {
t.Fatalf("prepared delivery was not invoked: %s (%v)", transcript, err)
}
})
}
}