Verify the reliability sprint and guard release checkout validation
This commit is contained in:
@@ -24,7 +24,8 @@ steps:
|
|||||||
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
git checkout --detach "$${tag_commit}"
|
git checkout --detach "$${tag_commit}"
|
||||||
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
test -z "$(git status --porcelain --untracked-files=all)"
|
checkout_status=$(git status --porcelain --untracked-files=all) || exit $?
|
||||||
|
test -z "$${checkout_status}"
|
||||||
|
|
||||||
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
|
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
|
||||||
trap 'rm -rf "$${bootstrap_dir}"' EXIT
|
trap 'rm -rf "$${bootstrap_dir}"' EXIT
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -10,11 +11,11 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
os.Exit(run())
|
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() int {
|
func run(args []string, stdout, stderr io.Writer) int {
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
return cli.Run(ctx, os.Args[1:], os.Stdout, os.Stderr)
|
return cli.Run(ctx, args, stdout, stderr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"os/signal"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -41,10 +40,7 @@ func TestProcessSubprocessHelper(t *testing.T) {
|
|||||||
}
|
}
|
||||||
syscall.Umask(int(mask))
|
syscall.Umask(int(mask))
|
||||||
}
|
}
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
os.Exit(run(os.Args[i+1:], os.Stdout, os.Stderr))
|
||||||
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
|
|
||||||
stop()
|
|
||||||
os.Exit(code)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
@@ -237,7 +233,6 @@ type processFixture struct {
|
|||||||
dir string
|
dir string
|
||||||
config string
|
config string
|
||||||
transcript string
|
transcript string
|
||||||
glossary string
|
|
||||||
output string
|
output string
|
||||||
provider *fixtureProvider
|
provider *fixtureProvider
|
||||||
server *httptest.Server
|
server *httptest.Server
|
||||||
@@ -282,11 +277,8 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
glossary := filepath.Join(dir, "glossary.yaml")
|
|
||||||
if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
return processFixture{dir: dir, config: configPath, transcript: transcript,
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary,
|
|
||||||
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -314,25 +306,6 @@ func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (
|
|||||||
return code, stdout.String(), stderr.String()
|
return code, stdout.String(), stderr.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f processFixture) runConfigView(t *testing.T, view string) (int, string, string) {
|
|
||||||
t.Helper()
|
|
||||||
command := exec.Command(os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--", "config", "print-effective", "--config", f.config, "--view", view)
|
|
||||||
command.Dir = f.dir
|
|
||||||
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
command.Stdout, command.Stderr = &stdout, &stderr
|
|
||||||
err := command.Run()
|
|
||||||
code := 0
|
|
||||||
if err != nil {
|
|
||||||
if exit, ok := err.(*exec.ExitError); ok {
|
|
||||||
code = exit.ExitCode()
|
|
||||||
} else {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return code, stdout.String(), stderr.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
||||||
@@ -879,7 +852,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
for _, selection := range []struct{ reporting, debug bool }{
|
for _, selection := range []struct{ reporting, debug bool }{
|
||||||
{false, false}, {false, true}, {true, false}, {true, true},
|
{false, false}, {false, true}, {true, false}, {true, true},
|
||||||
} {
|
} {
|
||||||
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
|
||||||
reportDir := filepath.Join(f.dir, "missing")
|
reportDir := filepath.Join(f.dir, "missing")
|
||||||
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
|
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
|
||||||
@@ -894,7 +867,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
extra = append(extra, "--debug")
|
extra = append(extra, "--debug")
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, true, extra...)
|
code, stdout, stderr := f.run(t, true, extra...)
|
||||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
|
||||||
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
@@ -1072,8 +1045,9 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
err := command.Wait()
|
err := command.Wait()
|
||||||
if ctx.Err() != nil || err == nil {
|
var exit *exec.ExitError
|
||||||
t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String())
|
if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 1 {
|
||||||
|
t.Fatalf("interrupt did not reach the failure exit: %v %v stderr=%q", err, ctx.Err(), stderr.String())
|
||||||
}
|
}
|
||||||
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
|
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
|
||||||
t.Fatalf("cancellation report: %+v", document)
|
t.Fatalf("cancellation report: %+v", document)
|
||||||
@@ -1556,11 +1530,13 @@ func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) {
|
|||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
t.Fatal("proposal progress did not reach stderr while the provider was blocked")
|
t.Fatal("proposal progress did not reach stderr while the provider was blocked")
|
||||||
}
|
}
|
||||||
if err := cmd.Process.Signal(os.Interrupt); err != nil {
|
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := cmd.Wait(); err == nil || ctx.Err() != nil {
|
err = cmd.Wait()
|
||||||
t.Fatalf("interrupt did not complete the invocation: %v", err)
|
var exit *exec.ExitError
|
||||||
|
if !errors.As(err, &exit) || exit.ExitCode() != 1 || ctx.Err() != nil {
|
||||||
|
t.Fatalf("termination did not reach the failure exit: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,8 @@ repair. An expired parent context stops processing and retains any native cause.
|
|||||||
With an active parent, PromptKit's operation deadline remains recoverable even
|
With an active parent, PromptKit's operation deadline remains recoverable even
|
||||||
when a preparation error wraps it. A native transport timeout affects the HTTP
|
when a preparation error wraps it. A native transport timeout affects the HTTP
|
||||||
attempt and is recorded as a transport failure. The prepared notification
|
attempt and is recorded as a transport failure. The prepared notification
|
||||||
provides the actual request snapshot; failures before notification have no snapshot.
|
supplies prompt metadata to observers. With debug capture enabled, it also
|
||||||
|
supplies the actual request messages when preparation reaches that point.
|
||||||
PromptKit owns backend and credential selection. When a required credential is absent,
|
PromptKit owns backend and credential selection. When a required credential is absent,
|
||||||
the CLI identifies the role, profile, and environment variable using the native
|
the CLI identifies the role, profile, and environment variable using the native
|
||||||
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)
|
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
# Migrating to Audita v2
|
# Migrating to Audita v2
|
||||||
|
|
||||||
|
This guide describes the historical v2 release. For current commands,
|
||||||
|
configuration, and example projects, use the [v3 migration guide](v3.md).
|
||||||
|
|
||||||
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
||||||
Version-1 configuration files, environment overlays, and retired flags are not
|
Version-1 configuration files, environment overlays, and retired flags are not
|
||||||
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or
|
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ record the mismatch explicitly. A test that asserts only nonzero status does
|
|||||||
not verify this fix.
|
not verify this fix.
|
||||||
|
|
||||||
**Complete.** The upstream handoff is recorded below. The exit-status fix
|
**Complete.** The upstream handoff is recorded below. The exit-status fix
|
||||||
remains outstanding for the final verification pass.
|
remains outstanding after the final verification pass.
|
||||||
|
|
||||||
Build a temporary executable with `go build -o /tmp/audita-inspection ./cmd/audita`.
|
Build a temporary executable with `go build -o /tmp/audita-inspection ./cmd/audita`.
|
||||||
In a scratch directory, put `version: 3` in `config.yml`, `profiles: [` in the
|
In a scratch directory, put `version: 3` in `config.yml`, `profiles: [` in the
|
||||||
@@ -747,3 +747,68 @@ Audit #28, #29, #58, #60–62 and sprint closure.
|
|||||||
The completed sprint should leave the target architecture reachable through
|
The completed sprint should leave the target architecture reachable through
|
||||||
`audita process` and the standard delegated commands. Report checks and
|
`audita process` and the standard delegated commands. Report checks and
|
||||||
measurements actually performed, including any outstanding external dependency.
|
measurements actually performed, including any outstanding external dependency.
|
||||||
|
|
||||||
|
**Complete.** The integrated Audita work and release bootstrap passed checks.
|
||||||
|
The delegated inspection exit status remains an outstanding upstream item.
|
||||||
|
|
||||||
|
The bootstrap rejects a dirty checkout and a failed status command with no
|
||||||
|
output. A clean fixture reaches ReleaseKit with the prepared checkout and
|
||||||
|
expected commit arguments. Process fixtures share the production entry point
|
||||||
|
and check interrupt and termination exit status. Registry coverage checks
|
||||||
|
effective roles, required prompt identities and preparation, schema identity,
|
||||||
|
and independent copies. The ordered processing fixture decodes debug artifact
|
||||||
|
coordinates. The full repair/filter/later-failure case remains separate from
|
||||||
|
the smaller output-choice matrix. Unused fixture helpers and the legacy
|
||||||
|
glossary input were removed.
|
||||||
|
|
||||||
|
### Delegated inspection status
|
||||||
|
|
||||||
|
The [CommandKit upstream tags](https://forge.maximumdirect.net/base-layer/commandkit/tags)
|
||||||
|
confirmed `v0.7.0` is the latest release of both `ckprompt` and `ckconfig`.
|
||||||
|
The built Audita executable reproduced the Stage 1 mismatch:
|
||||||
|
|
||||||
|
| Invalid input | Path | Observed status and diagnostic |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Malformed automatic `prompts.yml` | `process` | 2, `yaml.invalid` |
|
||||||
|
| Malformed automatic `prompts.yml` | `promptkit list prompts` | 1, `yaml.invalid` |
|
||||||
|
| Malformed automatic `prompts.yml` | `config show/sources/validate/explain/diff --scope promptkit` | 1, `yaml.invalid` |
|
||||||
|
| Malformed explicit app main file | `config show/sources/validate/explain/diff --scope app` | 1, `yaml.invalid` |
|
||||||
|
|
||||||
|
Caller cancellation remains status 1 in the shared CLI and subprocess tests.
|
||||||
|
The CommandKit integrations still own the fix to return status 2 for typed
|
||||||
|
operator-input failures. Audita retains its native delegations. The expected
|
||||||
|
upstream change and reproduction remain in the Stage 1 handoff.
|
||||||
|
|
||||||
|
### Catalog closure
|
||||||
|
|
||||||
|
The table identifies the implementing stages and exceptions without changing
|
||||||
|
the historical audit observations. Each stage records its verification above.
|
||||||
|
|
||||||
|
| Catalog findings | Implementing stages or disposition |
|
||||||
|
| --- | --- |
|
||||||
|
| #1, #6–8, #16, #26, #42 | Stage 4. Only cooperative cancellation boundaries from #6 were selected. |
|
||||||
|
| #2, #4 | No implementation. The existing catalog dispositions stand. |
|
||||||
|
| #3, #9, #63–65 | Stages 2 and 14. |
|
||||||
|
| #5, #57 | Stage 19. |
|
||||||
|
| #10–11, #27, #40, #49 | Stages 9, 11, and 17. |
|
||||||
|
| #12, #18–19, #38, #47 | Stage 8. Only punctuation-edge handling from #18 was selected. |
|
||||||
|
| #13, #31, #51–52, #54 | Stages 2 and 5. |
|
||||||
|
| #14–15, #22, #30–32, #39, #41 | Stages 5, 12, and 15–18. |
|
||||||
|
| #17, #21, #35, #53, #59, #62 | Stages 5 and 6, with final caller checks in Stage 20. |
|
||||||
|
| #20, #24 | Stages 7 and 11 for processing; Stage 1 and the upstream disposition above for delegated inspection. |
|
||||||
|
| #23, #25, #44 | Stages 12, 18, and 19. |
|
||||||
|
| #28–29, #58, #60–61 | Stage 20. |
|
||||||
|
| #33, #43 | Stages 3 and 18. |
|
||||||
|
| #34, #45, #48 | Stage 14. |
|
||||||
|
| #36 | Stage 10. |
|
||||||
|
| #37, #50, #55–56 | Stage 11. |
|
||||||
|
| #46 | Stage 13. |
|
||||||
|
|
||||||
|
Verified with Go 1.27.3 by building the CLI, running `go test -count=1 ./...`
|
||||||
|
offline, and running targeted race checks for the runner, LLM adapter,
|
||||||
|
validators, reports, diagnostics, glossary, and CLI. The final review
|
||||||
|
checked Go formatting, whitespace, and 278 relative documentation links
|
||||||
|
and exercised the example configurations. Measurements above cover merge
|
||||||
|
and glossary allocation reductions, validation context, repair request and
|
||||||
|
artifact costs, bounded section contention, and retained evidence by output
|
||||||
|
mode. These are synthetic measurements. No release was published.
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
commandkit "go.maximumdirect.net/commandkit"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
@@ -138,21 +137,63 @@ pipeline:
|
|||||||
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
||||||
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
||||||
}
|
}
|
||||||
markdown := string(reporting.Markdown(result.Snapshot))
|
type coordinates struct {
|
||||||
if !strings.Contains(markdown, `1: glossary (party/../\|<tag>)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
|
StageIndex int `json:"stage_index"`
|
||||||
t.Fatalf("report labels: %s", markdown)
|
StageID *string `json:"stage_id"`
|
||||||
}
|
}
|
||||||
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
|
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
|
||||||
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
|
check := func(name string, entries []coordinates) {
|
||||||
if readErr != nil {
|
t.Helper()
|
||||||
t.Fatal(readErr)
|
seen := make(map[int]bool)
|
||||||
|
for _, entry := range entries {
|
||||||
|
id, ok := expected[entry.StageIndex]
|
||||||
|
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
|
||||||
|
t.Fatalf("%s has wrong stage association: %+v", name, entry)
|
||||||
|
}
|
||||||
|
seen[entry.StageIndex] = true
|
||||||
}
|
}
|
||||||
var compact bytes.Buffer
|
if len(seen) != len(expected) {
|
||||||
if err := json.Compact(&compact, raw); err != nil {
|
t.Fatalf("%s missing stage associations: %v", name, seen)
|
||||||
t.Fatalf("decode %s: %v", artifact, err)
|
|
||||||
}
|
}
|
||||||
if text := compact.String(); !strings.Contains(text, `"stage_index":0`) || !strings.Contains(text, `"stage_id":"party/../|\u003ctag\u003e"`) || !strings.Contains(text, `"stage_index":1`) || !strings.Contains(text, `"stage_id":"spell"`) || !strings.Contains(text, `"stage_index":2`) {
|
}
|
||||||
t.Fatalf("%s missing attribution: %s", artifact, raw)
|
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var ledger []coordinates
|
||||||
|
if err := json.Unmarshal(raw, &ledger); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check("ledger", ledger)
|
||||||
|
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var calls struct {
|
||||||
|
Calls []struct {
|
||||||
|
Context struct {
|
||||||
|
coordinates
|
||||||
|
Role string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &calls); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var locations []coordinates
|
||||||
|
roles := make(map[int]map[string]bool)
|
||||||
|
for _, call := range calls.Calls {
|
||||||
|
location := call.Context.coordinates
|
||||||
|
locations = append(locations, location)
|
||||||
|
if roles[location.StageIndex] == nil {
|
||||||
|
roles[location.StageIndex] = make(map[string]bool)
|
||||||
|
}
|
||||||
|
roles[location.StageIndex][call.Context.Role] = true
|
||||||
|
}
|
||||||
|
check("LLM observations", locations)
|
||||||
|
for index := range expected {
|
||||||
|
if !roles[index]["proposal"] || !roles[index]["validation"] {
|
||||||
|
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
commandkit "go.maximumdirect.net/commandkit"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
@@ -23,7 +24,7 @@ func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
|
|||||||
return main
|
return main
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
|
func TestResourceCommandsRejectInvalidOperatorPrompts(t *testing.T) {
|
||||||
definitions := map[string]string{
|
definitions := map[string]string{
|
||||||
"malformed operator YAML": "profiles: [\n",
|
"malformed operator YAML": "profiles: [\n",
|
||||||
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
|
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
|
||||||
@@ -40,6 +41,28 @@ func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMalformedCompanionKeepsDelegatedExitStatusVisible(t *testing.T) {
|
||||||
|
main := writeRuntimeErrorConfig(t, "profiles: [\n")
|
||||||
|
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
|
||||||
|
if err := os.WriteFile(transcript, []byte("[]\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
args []string
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{[]string{"promptkit", "list", "prompts"}, 1},
|
||||||
|
{[]string{"config", "show", "--scope", "promptkit"}, 1},
|
||||||
|
{[]string{"process", transcript}, 2},
|
||||||
|
} {
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
code := Run(context.Background(), append(tc.args, "--config", main), &stdout, &stderr)
|
||||||
|
if code != tc.status || !strings.Contains(stderr.String(), "yaml.invalid") || stdout.Len() != 0 {
|
||||||
|
t.Fatalf("%v: status=%d want=%d stdout=%q stderr=%q", tc.args, code, tc.status, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
|
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
|
||||||
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
|
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
|
||||||
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
|
transcript := filepath.Join(filepath.Dir(main), "transcript.json")
|
||||||
|
|||||||
@@ -234,7 +234,11 @@ func recordLedgerDecision(decision runner.ValidatorDecisionRecord, deterministic
|
|||||||
ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName,
|
ArtifactPath: decision.DiagnosticArtifactPath, ValidatorKey: decision.ValidatorName,
|
||||||
Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message,
|
Approved: decision.Approved, Disposition: decision.Disposition, ReasonCode: decision.ReasonCode, Message: decision.Message,
|
||||||
}
|
}
|
||||||
if decision.ExecutionClass == validatormetadata.ExecutionClassLLMBacked {
|
class := decision.ExecutionClass
|
||||||
|
if class == "" {
|
||||||
|
class = validatormetadata.ClassForKey(decision.ValidatorName)
|
||||||
|
}
|
||||||
|
if class == validatormetadata.ExecutionClassLLMBacked {
|
||||||
*llm = append(*llm, record)
|
*llm = append(*llm, record)
|
||||||
} else {
|
} else {
|
||||||
*deterministic = append(*deterministic, record)
|
*deterministic = append(*deterministic, record)
|
||||||
|
|||||||
@@ -68,17 +68,6 @@ func TestLookupUnknownSchema(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegisteredSchemasLoadReadableEmbeddedJSON(t *testing.T) {
|
|
||||||
for _, schema := range Registered() {
|
|
||||||
if len(schema.JSONSchema) == 0 {
|
|
||||||
t.Fatalf("expected non-empty JSON schema for %q", schema.ID)
|
|
||||||
}
|
|
||||||
if !json.Valid(schema.JSONSchema) {
|
|
||||||
t.Fatalf("expected valid JSON schema for %q", schema.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
func TestSchemaHashesMatchRegisteredJSON(t *testing.T) {
|
||||||
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
|
for _, key := range []Key{CorrectionSetKey, CorrectionRepairKey, ValidatorDecisionSetKey} {
|
||||||
schema, ok := Lookup(key)
|
schema, ok := Lookup(key)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -451,34 +450,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.After(350 * time.Millisecond)
|
|
||||||
got := 0
|
|
||||||
for got < want {
|
|
||||||
select {
|
|
||||||
case <-entered:
|
|
||||||
got++
|
|
||||||
case <-deadline:
|
|
||||||
t.Fatalf("timed out waiting for %d entries (%s), got %d", want, label, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func waitForRunnerSectionIDs(t *testing.T, started <-chan int, want map[int]struct{}) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.After(350 * time.Millisecond)
|
|
||||||
seen := map[int]struct{}{}
|
|
||||||
for len(seen) < len(want) {
|
|
||||||
select {
|
|
||||||
case id := <-started:
|
|
||||||
seen[id] = struct{}{}
|
|
||||||
case <-deadline:
|
|
||||||
t.Fatalf("timed out waiting for section IDs %v, got %v", mapKeys(want), mapKeys(seen))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapKeys(values map[int]struct{}) []int {
|
func mapKeys(values map[int]struct{}) []int {
|
||||||
keys := make([]int, 0, len(values))
|
keys := make([]int, 0, len(values))
|
||||||
for key := range values {
|
for key := range values {
|
||||||
@@ -563,60 +534,6 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type stageAwareStructuredClient struct {
|
|
||||||
startedSection chan int
|
|
||||||
releaseValidation <-chan struct{}
|
|
||||||
eventSink chan<- string
|
|
||||||
}
|
|
||||||
|
|
||||||
type captureContractStructuredClient struct {
|
|
||||||
lastRequest contracts.StructuredCompletionRequest
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
||||||
_ = ctx
|
|
||||||
c.lastRequest = req
|
|
||||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
|
||||||
*target = validators.LLMValidationResponse{
|
|
||||||
Validations: []validators.LLMValidationDecision{
|
|
||||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return contracts.StructuredCompletionResponse{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
||||||
section := 0
|
|
||||||
if req.Context.SectionIndex != nil {
|
|
||||||
section = *req.Context.SectionIndex
|
|
||||||
}
|
|
||||||
if c.startedSection != nil {
|
|
||||||
select {
|
|
||||||
case c.startedSection <- section:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if c.eventSink != nil {
|
|
||||||
c.eventSink <- "v" + strconv.Itoa(section)
|
|
||||||
}
|
|
||||||
if c.releaseValidation != nil {
|
|
||||||
select {
|
|
||||||
case <-c.releaseValidation:
|
|
||||||
case <-ctx.Done():
|
|
||||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
target := out.(*validators.LLMValidationResponse)
|
|
||||||
*target = validators.LLMValidationResponse{
|
|
||||||
Validations: []validators.LLMValidationDecision{
|
|
||||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return contracts.StructuredCompletionResponse{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunnerSkippedRecorded(t *testing.T) {
|
func TestRunnerSkippedRecorded(t *testing.T) {
|
||||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "word word"}}}
|
||||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||||
|
|||||||
@@ -76,19 +76,27 @@ func TestEmbeddedDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRegistryMetadataMatchesEmbeddedDefinitions(t *testing.T) {
|
func TestRegistryMetadataIdentifiesRequiredPrompts(t *testing.T) {
|
||||||
all := RegisteredMetadata()
|
required := []string{
|
||||||
if len(all) != 11 {
|
"audita/modules.glossary.proposal", "audita/modules.glossary.repair",
|
||||||
t.Fatalf("prompt count: %d", len(all))
|
"audita/modules.grammar.proposal", "audita/modules.grammar.repair",
|
||||||
|
"audita/modules.homophones.proposal", "audita/modules.homophones.repair",
|
||||||
|
"audita/modules.spoken_word.proposal", "audita/modules.spoken_word.repair",
|
||||||
|
"audita/validators.editorial_review", "audita/validators.meaning_reversal_review",
|
||||||
|
"audita/validators.spoken_form_plausibility",
|
||||||
}
|
}
|
||||||
for _, meta := range all {
|
for _, id := range required {
|
||||||
if meta.PromptSource != SourceBuiltin || meta.EmbeddedPath == "" {
|
metadata, ok := LookupMetadata(id)
|
||||||
t.Fatalf("metadata: %+v", meta)
|
if !ok || metadata.PromptID != id || metadata.PromptSource != SourceBuiltin ||
|
||||||
}
|
metadata.EmbeddedPath != AssetRoot+"/"+strings.TrimPrefix(id, "audita/")+".yaml" {
|
||||||
if _, err := Filesystem().Open(meta.EmbeddedPath); err != nil {
|
t.Fatalf("required prompt metadata for %q: %+v", id, metadata)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
all := RegisteredMetadata()
|
||||||
|
all[0].PromptID = "changed"
|
||||||
|
if again := RegisteredMetadata(); again[0].PromptID == "changed" {
|
||||||
|
t.Fatal("metadata enumeration shares mutable state")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
func TestRepairDefinitionsPrepareThroughPromptKit(t *testing.T) {
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
package testsupport
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ReadFile(t *testing.T, path string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func OnlyRunDir(t *testing.T, workDir string) string {
|
|
||||||
t.Helper()
|
|
||||||
entries, err := os.ReadDir(workDir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read work dir %q: %v", workDir, err)
|
|
||||||
}
|
|
||||||
dirs := make([]string, 0, len(entries))
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.IsDir() {
|
|
||||||
dirs = append(dirs, filepath.Join(workDir, e.Name()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(dirs) != 1 {
|
|
||||||
t.Fatalf("expected exactly one run dir in %q, found %d", workDir, len(dirs))
|
|
||||||
}
|
|
||||||
return dirs[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
func AssertNoSecretInFile(t *testing.T, path, secret string) {
|
|
||||||
t.Helper()
|
|
||||||
raw := string(ReadFile(t, path))
|
|
||||||
if strings.Contains(raw, secret) {
|
|
||||||
t.Fatalf("secret leaked in %s", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func AssertNoSecretInTree(t *testing.T, root, secret string) {
|
|
||||||
t.Helper()
|
|
||||||
_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil || d == nil || d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
raw, readErr := os.ReadFile(path)
|
|
||||||
if readErr == nil && strings.Contains(string(raw), secret) {
|
|
||||||
t.Fatalf("secret leaked in %s", path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -9,104 +9,33 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
|
|
||||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/no_effect"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/non_empty_resulting_segment"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/original_text_presence"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/proposal_shape"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/spoken_form_plausibility"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuiltInRegistryRegistersAllKeys(t *testing.T) {
|
func TestRegisteredValidatorsUseDocumentedRoles(t *testing.T) {
|
||||||
r := NewBuiltInRegistry()
|
registry := NewBuiltInRegistry()
|
||||||
for _, key := range []string{
|
for _, tc := range []struct {
|
||||||
KeyProposalShape,
|
key string
|
||||||
KeyOriginalTextPresence,
|
role validatormetadata.ExecutionClass
|
||||||
KeyNonEmptyResultingSegment,
|
}{
|
||||||
KeyNoEffect,
|
{KeyProposalShape, validatormetadata.ExecutionClassDeterministic},
|
||||||
KeyProtectedTerms,
|
{KeyOriginalTextPresence, validatormetadata.ExecutionClassDeterministic},
|
||||||
KeySpokenFormPlausibility,
|
{KeyNonEmptyResultingSegment, validatormetadata.ExecutionClassDeterministic},
|
||||||
KeyMeaningReversalReview,
|
{KeyNoEffect, validatormetadata.ExecutionClassDeterministic},
|
||||||
KeyEditorialReview,
|
{KeyProtectedTerms, validatormetadata.ExecutionClassDeterministic},
|
||||||
|
{KeySpokenFormPlausibility, validatormetadata.ExecutionClassLLMBacked},
|
||||||
|
{KeyMeaningReversalReview, validatormetadata.ExecutionClassLLMBacked},
|
||||||
|
{KeyEditorialReview, validatormetadata.ExecutionClassLLMBacked},
|
||||||
} {
|
} {
|
||||||
def, ok := r.Lookup(key)
|
t.Run(tc.key, func(t *testing.T) {
|
||||||
if !ok {
|
v, err := registry.MustBuild(tc.key)
|
||||||
t.Fatalf("expected validator key %q to be registered", key)
|
if err != nil {
|
||||||
}
|
t.Fatal(err)
|
||||||
v, err := def.Build()
|
}
|
||||||
if err != nil {
|
if v.Name() != tc.key || validatormetadata.ClassOf(v) != tc.role {
|
||||||
t.Fatalf("build validator %q: %v", key, err)
|
t.Fatalf("registered name=%q role=%q, want %q and %q", v.Name(), validatormetadata.ClassOf(v), tc.key, tc.role)
|
||||||
}
|
}
|
||||||
if v.Name() != key {
|
})
|
||||||
t.Fatalf("expected validator name %q, got %q", key, v.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
|
||||||
type validatorCtor struct {
|
|
||||||
name string
|
|
||||||
key string
|
|
||||||
build func() (frameworkvalidators.Validator, error)
|
|
||||||
wantClass validatormetadata.ExecutionClass
|
|
||||||
}
|
|
||||||
cases := []validatorCtor{
|
|
||||||
{name: "proposal_shape", key: KeyProposalShape, build: proposal_shape.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
|
||||||
{name: "original_text_presence", key: KeyOriginalTextPresence, build: original_text_presence.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
|
||||||
{name: "non_empty_resulting_segment", key: KeyNonEmptyResultingSegment, build: non_empty_resulting_segment.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
|
||||||
{name: "no_effect", key: KeyNoEffect, build: no_effect.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
|
||||||
{name: "protected_terms", key: KeyProtectedTerms, build: protected_terms.New, wantClass: validatormetadata.ExecutionClassDeterministic},
|
|
||||||
{name: "spoken_form_plausibility", key: KeySpokenFormPlausibility, build: spoken_form_plausibility.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
|
|
||||||
{name: "meaning_reversal_review", key: KeyMeaningReversalReview, build: meaning_reversal_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
|
|
||||||
{name: "editorial_review", key: KeyEditorialReview, build: editorial_review.New, wantClass: validatormetadata.ExecutionClassLLMBacked},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
v, err := tc.build()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("%s: build: %v", tc.name, err)
|
|
||||||
}
|
|
||||||
if v.Name() != tc.key {
|
|
||||||
t.Fatalf("%s: expected key %q, got %q", tc.name, tc.key, v.Name())
|
|
||||||
}
|
|
||||||
if got := validatormetadata.ClassOf(v); got != tc.wantClass {
|
|
||||||
t.Fatalf("%s: expected class %q, got %q", tc.name, tc.wantClass, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistryBuildsClassifiedValidators(t *testing.T) {
|
|
||||||
r := NewBuiltInRegistry()
|
|
||||||
for _, key := range r.RegisteredKeys() {
|
|
||||||
v, err := r.MustBuild(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("must build %q: %v", key, err)
|
|
||||||
}
|
|
||||||
if _, ok := v.(validatormetadata.ClassifiedValidator); !ok {
|
|
||||||
t.Fatalf("expected built validator %q to expose execution classification metadata", key)
|
|
||||||
}
|
|
||||||
want := validatormetadata.ClassForKey(key)
|
|
||||||
if got := validatormetadata.ClassOf(v); got != want {
|
|
||||||
t.Fatalf("expected class %q for %q, got %q", want, key, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) {
|
|
||||||
r := NewBuiltInRegistry()
|
|
||||||
for _, key := range r.RegisteredKeys() {
|
|
||||||
v, err := r.MustBuild(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("must build %q: %v", key, err)
|
|
||||||
}
|
|
||||||
fromKey := validatormetadata.ClassForKey(key)
|
|
||||||
fromInstance := validatormetadata.ClassOf(v)
|
|
||||||
if fromInstance != fromKey {
|
|
||||||
t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,12 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"maps"
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) {
|
func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) {
|
||||||
@@ -60,27 +65,6 @@ func TestReleaseKitProfileDeclaresProductAndDeliverySettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReleaseKitTargetMembership(t *testing.T) {
|
|
||||||
want := []string{"linux/amd64", "linux/arm64"}
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
targets []string
|
|
||||||
match bool
|
|
||||||
}{
|
|
||||||
{"missing amd64", []string{"linux/arm64", "linux/arm64"}, false},
|
|
||||||
{"missing arm64", []string{"linux/amd64", "linux/amd64"}, false},
|
|
||||||
{"additional target", []string{"linux/amd64", "linux/arm64", "darwin/arm64"}, false},
|
|
||||||
{"reordered", []string{"linux/arm64", "linux/amd64"}, true},
|
|
||||||
{"repeated", []string{"linux/amd64", "linux/arm64", "linux/amd64"}, true},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
if got := sameMembers(test.targets, want); got != test.match {
|
|
||||||
t.Fatalf("target membership for %q = %t, want %t", test.targets, got, test.match)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func sameMembers(got, want []string) bool {
|
func sameMembers(got, want []string) bool {
|
||||||
gotSet := make(map[string]struct{}, len(got))
|
gotSet := make(map[string]struct{}, len(got))
|
||||||
for _, value := range got {
|
for _, value := range got {
|
||||||
@@ -92,3 +76,106 @@ func sameMembers(got, want []string) bool {
|
|||||||
}
|
}
|
||||||
return maps.Equal(gotSet, wantSet)
|
return maps.Equal(gotSet, wantSet)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReleaseBootstrapRequiresSuccessfulCleanStatus(t *testing.T) {
|
||||||
|
data, err := os.ReadFile("../.woodpecker/release.yml")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var workflow struct {
|
||||||
|
Steps []struct {
|
||||||
|
Name string
|
||||||
|
Commands []string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := yaml.Unmarshal(data, &workflow); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var script string
|
||||||
|
for _, step := range workflow.Steps {
|
||||||
|
if step.Name == "deliver" {
|
||||||
|
// Woodpecker removes one dollar from escaped shell variables.
|
||||||
|
script = strings.ReplaceAll(strings.Join(step.Commands, "\n"), "$${", "${")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if script == "" {
|
||||||
|
t.Fatal("delivery command is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status string
|
||||||
|
exit int
|
||||||
|
}{
|
||||||
|
{"clean", "clean", 0},
|
||||||
|
{"dirty", "dirty", 1},
|
||||||
|
{"status failed with no output", "failed", 93},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
workspace := filepath.Join(dir, "checkout")
|
||||||
|
if err := os.Mkdir(workspace, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(workspace, "go.mod"), []byte("module example.invalid/release\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writeCommand := func(name, script string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("#!/bin/sh\nset -eu\n"+script), 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeCommand("git", `
|
||||||
|
case "$1" in
|
||||||
|
check-ref-format|fetch|checkout) ;;
|
||||||
|
rev-parse) printf '%s\n' "$CI_COMMIT_SHA" ;;
|
||||||
|
status)
|
||||||
|
test "$*" = 'status --porcelain --untracked-files=all'
|
||||||
|
case "$STATUS_FIXTURE" in
|
||||||
|
dirty) printf '%s\n' '?? untracked' ;;
|
||||||
|
failed) exit 93 ;;
|
||||||
|
esac
|
||||||
|
;;
|
||||||
|
*) exit 99 ;;
|
||||||
|
esac
|
||||||
|
`)
|
||||||
|
writeCommand("go", `
|
||||||
|
case "$*" in
|
||||||
|
'env GOHOSTOS') printf '%s\n' linux ;;
|
||||||
|
'env GOHOSTARCH') printf '%s\n' amd64 ;;
|
||||||
|
esac
|
||||||
|
printf '%s\n' "$*" >> "$GO_CALLS_FIXTURE"
|
||||||
|
`)
|
||||||
|
calls := filepath.Join(dir, "go-calls")
|
||||||
|
commit := strings.Repeat("a", 40)
|
||||||
|
command := exec.Command("sh", "-c", script)
|
||||||
|
command.Dir = workspace
|
||||||
|
command.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"),
|
||||||
|
"CI_WORKSPACE="+workspace, "CI_COMMIT_TAG=v1.2.3", "CI_COMMIT_SHA="+commit,
|
||||||
|
"STATUS_FIXTURE="+tc.status, "GO_CALLS_FIXTURE="+calls)
|
||||||
|
output, err := command.CombinedOutput()
|
||||||
|
exit := 0
|
||||||
|
if err != nil {
|
||||||
|
failure, ok := err.(*exec.ExitError)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exit = failure.ExitCode()
|
||||||
|
}
|
||||||
|
if exit != tc.exit {
|
||||||
|
t.Fatalf("exit=%d want %d: %s", exit, tc.exit, output)
|
||||||
|
}
|
||||||
|
transcript, err := os.ReadFile(calls)
|
||||||
|
if tc.exit != 0 {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("bootstrap reached Go after failed status: %s (%v)", transcript, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || !strings.Contains(string(transcript), "ci-deliver --repo "+workspace+" --prepared --commit "+commit+" v1.2.3") {
|
||||||
|
t.Fatalf("prepared delivery was not invoked: %s (%v)", transcript, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user