Compare commits
260 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| da5b8913d1 | |||
| ea4e82e380 | |||
| 8551121ef6 | |||
| 788f830b19 | |||
| 3e107bedd1 | |||
| 19755e1d47 | |||
| b3ed91d593 | |||
| cc0a420722 | |||
| ee1ac8fc89 | |||
| 7f7aea0921 | |||
| 743e64a07a | |||
| d26991f0bf | |||
| 44c767f285 | |||
| 2d93e8ae7c | |||
| 9ad79ce13b | |||
| 9192239bfd | |||
| 1bc54a405c | |||
| fd65d9ca1e | |||
| a88141ef4a | |||
| c6be2ec20d | |||
| 0c2a2164ec | |||
| f99fd993b4 | |||
| 1c90fedb1e | |||
| 087e388a03 | |||
| e8c070805e | |||
| a355f05a27 | |||
| 11f314d995 | |||
| 891d4aac2c | |||
| 5167154a1c | |||
| b98d463958 | |||
| 1c43264f6f | |||
| e2710d869e | |||
| a0c70e8dce | |||
| c2c8050bc8 | |||
| e98e525b5c | |||
| b1d1b8302c | |||
| 0c144fd5a5 | |||
| 475557e4de | |||
| 43ce620122 | |||
| 27f494d4e4 | |||
| f465496dd8 | |||
| b63c8af467 | |||
| cea7e44232 | |||
| ffe0e3491e | |||
| e34672f7fe | |||
| d226bdd697 | |||
| 1d32e9bad5 | |||
| 730c1e4758 | |||
| a23ce14285 | |||
| 073e6cea26 | |||
| 34da87f439 | |||
| 571ad70a54 | |||
| 176f59b176 | |||
| 747a66999f | |||
| a68a951f25 | |||
| 42e23f8563 | |||
| faba8abf17 | |||
| 72cc24030e | |||
| fb6cb49ba6 | |||
| 9110aedcd7 | |||
| 0f34231f46 | |||
| 7d064a6d5a | |||
| 416cdea625 | |||
| 27e58fabf0 | |||
| 4ecb71c741 | |||
| e4e5dd74e3 | |||
| c0193df387 | |||
| 717d59c3b0 | |||
| 0eca7b85d8 | |||
| c4ccd7897e | |||
| e183708021 | |||
| 5277a1ca19 | |||
| 6124162c7b | |||
| 0ef7aca294 | |||
| dfdc6cc2c4 | |||
| d6413b3996 | |||
| fc607cfdf0 | |||
| d98d419b36 | |||
| d28b560265 | |||
| f42573f8b0 | |||
| cf1fbe5748 | |||
| d155bf4cb3 | |||
| bbf89bb4f5 | |||
| 7c567d8d8b | |||
| d15d80c79f | |||
| 2712a3887f | |||
| 6e30054c36 | |||
| 2261a38a6b | |||
| 2b5b0d6051 | |||
| 03d1884bfb | |||
| 5aca3ebccf | |||
| 6a843481d2 | |||
| d2eb5a5662 | |||
| 2be7bbc015 | |||
| 64e5d0d275 | |||
| ca36c3e319 | |||
| 6d34602f0a | |||
| 891c204e8f | |||
| 73ef2c6083 | |||
| 04e0c2f913 | |||
| 6b724fd3cc | |||
| 8c93f728c0 | |||
| 5d90478f70 | |||
| 45b42a2689 | |||
| 8267eba6a5 | |||
| bcce4729e1 | |||
| d3a9edb29d | |||
| d3fbddce11 | |||
| e029730c79 | |||
| aa8073cab1 | |||
| f4c638313e | |||
| 735dfc1834 | |||
| d927c61199 | |||
| a5003a1e52 | |||
| 9fd36001b4 | |||
| 00848203d7 | |||
| eb45d0b623 | |||
| bb81354e21 | |||
| 93a080059d | |||
| b036336e2a | |||
| 4f525e07cc | |||
| d91bb658c9 | |||
| 8946a82407 | |||
| 791deca3cd | |||
| c86f4fa416 | |||
| 2140314d40 | |||
| 71a9b1f560 | |||
| a9b7c37a7e | |||
| c16d61e091 | |||
| 8ad27e8368 | |||
| 1fd95abe95 | |||
| bb63c2cb10 | |||
| 7710101da0 | |||
| 50940083c8 | |||
| 8a2a19ccd3 | |||
| bbac312b32 | |||
| 56d39fca80 | |||
| 9a52c8c246 | |||
| a0e45c8f92 | |||
| bae96e5dd5 | |||
| 803effc6ea | |||
| 45e5b381f5 | |||
| a8eb5aea4d | |||
| af0f9ac960 | |||
| 3eacb33eb0 | |||
| 88e166e3b1 | |||
| fa4f891689 | |||
| 2844ce7e6a | |||
| ba1d5d2367 | |||
| 926fa26a31 | |||
| 7efa07ee9d | |||
| 89de2a7014 | |||
| d42e2095d7 | |||
| 0a21ee2b50 | |||
| 199c1d1dbe | |||
| 852e79405f | |||
| 8e1e073731 | |||
| 8cc298d092 | |||
| 50257ac554 | |||
| 497a1a18c5 | |||
| 7c64592cab | |||
| b500b855c6 | |||
| 40420a217a | |||
| 0c7622375f | |||
| 4920393204 | |||
| 866e4b9b23 | |||
| 547be2d9ab | |||
| f1fced498a | |||
| 97f6f48098 | |||
| 267aca4b76 | |||
| d539c06f45 | |||
| 5c69377563 | |||
| 098401f674 | |||
| f016471880 | |||
| 7e9bf6f149 | |||
| 77995b7a2c | |||
| 1b76a426cc | |||
| 607fefac47 | |||
| 6908dcd206 | |||
| 42930d4921 | |||
| 64786140e8 | |||
| af6f62e14b | |||
| 167d91131e | |||
| 4019f57eaa | |||
| 760a7cecc0 | |||
| 943152d1d5 | |||
| c94086048a | |||
| d2c0e22a9a | |||
| 0b3734d352 | |||
| b7bad87659 | |||
| c46a689e29 | |||
| 9dba2d506a | |||
| 68e16f6aac | |||
| 1f2beef08c | |||
| 32bac58b5e | |||
| 90d82b5b5f | |||
| a9dba890ba | |||
| 04f9f16e27 | |||
| 893f66b34b | |||
| 98ffb67432 | |||
| ec8dffc06e | |||
| d4d510a520 | |||
| ac86c77cfe | |||
| 259f728f20 | |||
| 46765bd326 | |||
| cbcb1b3778 | |||
| 39052bf47e | |||
| 7cd5edd650 | |||
| 2d41748006 | |||
| 1d4be9988e | |||
| ed65aa6474 | |||
| 4c5536ab78 | |||
| b01180ea83 | |||
| 07fab5586b | |||
| 6debe7e641 | |||
| 11c349d9ae | |||
| 36823c91fb | |||
| 17421bc5f3 | |||
| bd62f360a7 | |||
| 5a11b34aad | |||
| f9cf5c3861 | |||
| ea223fcf98 | |||
| 8d416558a3 | |||
| 561a84fcf8 | |||
| 3c405f0524 | |||
| 5ac704c4c4 | |||
| 5af6ba7d89 | |||
| 06dc273896 | |||
| 7e7940db9a | |||
| 721365e59d | |||
| 418da456d7 | |||
| 6de470b53d | |||
| 3b4933256e | |||
| 6f7136724f | |||
| 9bcf18331e | |||
| 5525fe875e | |||
| b81bd28865 | |||
| 9b82eb7da2 | |||
| f8d00e2f20 | |||
| f5fa607c35 | |||
| 43a68e97ee | |||
| b1b0fc7455 | |||
| afcf58331f | |||
| b31955fc97 | |||
| 1777821e8b | |||
| 79d38606fe | |||
| 8cba751835 | |||
| 8175ed87d0 | |||
| b64010e729 | |||
| 1cfe0c5a01 | |||
| 0e2c0b0b82 | |||
| cd9ee78359 | |||
| 23e1df4cfb | |||
| 2023c91ec3 | |||
| 67519051f1 | |||
| 6ae1d7cd5f | |||
| ddfc3dd4c7 | |||
| 990ca0b71a | |||
| 61bc467a03 | |||
| 0bd116b6b7 |
@@ -1,26 +1,52 @@
|
|||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- GO_IMAGE: golang:1.27.4-bookworm
|
||||||
|
|
||||||
when:
|
when:
|
||||||
- event: tag
|
- event: tag
|
||||||
|
ref: refs/tags/v*
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: build-release-assets
|
- name: deliver
|
||||||
image: golang:1.25.5
|
image: ${GO_IMAGE}
|
||||||
commands:
|
environment:
|
||||||
- bash scripts/build-release.sh "$CI_COMMIT_TAG" dist
|
RELEASEKIT_GITEA_TOKEN:
|
||||||
|
|
||||||
- name: publish-release
|
|
||||||
image: woodpeckerci/plugin-release
|
|
||||||
depends_on:
|
|
||||||
- build-release-assets
|
|
||||||
settings:
|
|
||||||
api_key:
|
|
||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
commands:
|
||||||
- dist/audita-*-linux-amd64
|
- |
|
||||||
- dist/audita-*-linux-arm64
|
set -eu
|
||||||
note: docs/releases/${CI_COMMIT_TAG}.md
|
test -n "$${CI_COMMIT_TAG}"
|
||||||
checksum: sha256
|
test -n "$${CI_COMMIT_SHA}"
|
||||||
checksum-file: SHA256SUMS
|
git check-ref-format "refs/tags/$${CI_COMMIT_TAG}"
|
||||||
checksum-flatten: true
|
git fetch origin "refs/tags/$${CI_COMMIT_TAG}:refs/tags/$${CI_COMMIT_TAG}"
|
||||||
file-exists: skip
|
tag_commit=$(git rev-parse "refs/tags/$${CI_COMMIT_TAG}^{commit}")
|
||||||
overwrite: false
|
expected_commit=$(git rev-parse --verify "$${CI_COMMIT_SHA}^{commit}")
|
||||||
prerelease: false
|
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
|
git checkout --detach "$${tag_commit}"
|
||||||
|
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
|
||||||
|
checkout_status=$(git status --porcelain --untracked-files=all) || exit $?
|
||||||
|
test -z "$${checkout_status}"
|
||||||
|
|
||||||
|
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
|
||||||
|
trap 'rm -rf "$${bootstrap_dir}"' EXIT
|
||||||
|
cp go.mod "$${bootstrap_dir}/releasekit.mod"
|
||||||
|
if [ -f go.sum ]; then
|
||||||
|
cp go.sum "$${bootstrap_dir}/releasekit.sum"
|
||||||
|
else
|
||||||
|
: > "$${bootstrap_dir}/releasekit.sum"
|
||||||
|
fi
|
||||||
|
|
||||||
|
export GOWORK=off
|
||||||
|
export GOTOOLCHAIN=local
|
||||||
|
releasekit_host_os=$(go env GOHOSTOS) || exit $?
|
||||||
|
releasekit_host_arch=$(go env GOHOSTARCH) || exit $?
|
||||||
|
export GOOS="$${releasekit_host_os}" GOARCH="$${releasekit_host_arch}"
|
||||||
|
releasekit_module=go.maximumdirect.net/releasekit
|
||||||
|
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -m "$${releasekit_module}"
|
||||||
|
go mod edit -modfile="$${bootstrap_dir}/releasekit.mod" -tool="$${releasekit_module}/cmd/releasekit"
|
||||||
|
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -deps "$${releasekit_module}/cmd/releasekit" > /dev/null
|
||||||
|
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
|
||||||
|
--repo "$${CI_WORKSPACE}" \
|
||||||
|
--prepared \
|
||||||
|
--commit "$${expected_commit}" \
|
||||||
|
"$${CI_COMMIT_TAG}"
|
||||||
|
|||||||
49
README.md
49
README.md
@@ -1,55 +1,34 @@
|
|||||||
# Audita
|
# Audita
|
||||||
|
|
||||||
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
|
Audita polishes transcript JSON with glossary-aware and LLM-backed correction
|
||||||
modules. It validates proposed corrections and produces corrected text plus
|
modules. It validates proposed edits and produces corrected text and reports for
|
||||||
reports for review and subprocess integration.
|
review or subprocess integration.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
From a checkout, build with Go 1.25.5 or newer:
|
Build with Go 1.27.0 or newer:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go build -o ./bin/audita ./cmd/audita
|
go build -o ./bin/audita ./cmd/audita
|
||||||
```
|
```
|
||||||
|
|
||||||
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
|
Edit the synthetic endpoint and model in [examples/local/prompts.yml](examples/local/prompts.yml)
|
||||||
and access to a model you select. The [minimal config](examples/minimal-config.yml)
|
for a reachable OpenAI-compatible service. From `examples/local`, run the built
|
||||||
targets an unauthenticated local endpoint; edit its endpoint and
|
executable (adjust the executable path if needed):
|
||||||
[local profiles](examples/profiles/) to match your service and models. The
|
|
||||||
[service example](examples/production-config.yml) demonstrates a named credential
|
|
||||||
reference and a shared backend. Set its referenced environment variable before
|
|
||||||
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
|
|
||||||
role profiles, but model cost, speed, and quality depend on your provider choices.
|
|
||||||
|
|
||||||
Check configuration and profile selection offline:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./bin/audita config validate --config examples/minimal-config.yml
|
../../bin/audita config validate --config config.yml
|
||||||
./bin/audita config print-effective --config examples/minimal-config.yml --view profiles
|
../../bin/audita process --config config.yml --output corrected.json transcript.json
|
||||||
```
|
```
|
||||||
|
|
||||||
Run the included small input with the minimal config:
|
Processing makes model requests. Reports are written beside the output by default;
|
||||||
|
recoverable model errors can appear in a completed run. The [example guide](examples/README.md)
|
||||||
```sh
|
includes service, overlay, prompt editing, preview, and shared glossary projects.
|
||||||
./bin/audita process examples/tiny-transcript.json \
|
See the [v3 migration guide](docs/migrations/v3.md) for the configuration cutover.
|
||||||
--config examples/minimal-config.yml \
|
|
||||||
--glossary examples/tiny-glossary.yaml \
|
|
||||||
--output corrected.json
|
|
||||||
```
|
|
||||||
|
|
||||||
This makes LLM requests. Explicit setting flags override the selected file and
|
|
||||||
optional overlay; general environment setting overrides are no longer read.
|
|
||||||
By default the command writes `errors.yml`, `recoveries.yml`, and `report.md`
|
|
||||||
beside `corrected.json`. Use `--reporting=false` to suppress those files, or
|
|
||||||
`--debug` to retain detailed run artifacts under the configured debug work
|
|
||||||
directory. Model failures affecting some operations can still yield a completed
|
|
||||||
transcript with applied corrections; the summary and reports identify them.
|
|
||||||
See the [v2 migration guide](docs/migrations/v2.md) for old configurations.
|
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md)
|
- [CLI reference](docs/cli.md) and [configuration](docs/config.md)
|
||||||
- [Configuration and maintained examples](docs/config.md)
|
|
||||||
- [Operations and artifact lifecycle](docs/operations.md)
|
- [Operations and artifact lifecycle](docs/operations.md)
|
||||||
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
|
||||||
- [Output files and report formats](docs/integrations/output-files.md)
|
- [Output files and report formats](docs/integrations/output-files.md)
|
||||||
|
|||||||
405
cmd/audita/correction_repair_integration_test.go
Normal file
405
cmd/audita/correction_repair_integration_test.go
Normal file
@@ -0,0 +1,405 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/reporting"
|
||||||
|
)
|
||||||
|
|
||||||
|
type repairWireMessage struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
type repairWireRequest struct {
|
||||||
|
Messages []repairWireMessage `json:"messages"`
|
||||||
|
ResponseFormat struct {
|
||||||
|
JSONSchema struct {
|
||||||
|
Schema struct {
|
||||||
|
Properties map[string]any `json:"properties"`
|
||||||
|
} `json:"schema"`
|
||||||
|
} `json:"json_schema"`
|
||||||
|
} `json:"response_format"`
|
||||||
|
}
|
||||||
|
type correctionProvider struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
requests []repairWireRequest
|
||||||
|
attempts map[int]int
|
||||||
|
entered chan struct{}
|
||||||
|
wait bool
|
||||||
|
invalidFirstRevision bool
|
||||||
|
decimalReferences bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req repairWireRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
http.Error(w, "bad request", 400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, repair := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]
|
||||||
|
_, validation := req.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
||||||
|
p.mu.Lock()
|
||||||
|
p.requests = append(p.requests, req)
|
||||||
|
decimalReferences := p.decimalReferences
|
||||||
|
p.mu.Unlock()
|
||||||
|
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello"},{"id":2,"original_text":"second","corrected_text":"Second"},{"id":3,"original_text":"third","corrected_text":"Third"}]}`
|
||||||
|
if validation {
|
||||||
|
var candidates []fixtureValidationItem
|
||||||
|
for _, message := range req.Messages {
|
||||||
|
if items, ok := fixtureValidationItems(message.Content); ok {
|
||||||
|
candidates = append(candidates, items...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var decisions []map[string]any
|
||||||
|
for _, item := range candidates {
|
||||||
|
p.mu.Lock()
|
||||||
|
approved := item.OriginalText != "third" && (item.OriginalText != "second" || p.attempts[2] > 1)
|
||||||
|
p.mu.Unlock()
|
||||||
|
reason := ""
|
||||||
|
if !approved {
|
||||||
|
reason = "The capitalization is unsupported in the supplied context."
|
||||||
|
}
|
||||||
|
decisions = append(decisions, map[string]any{"correction_index": item.CorrectionIndex, "approved": approved, "reason": reason})
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||||
|
content = string(raw)
|
||||||
|
}
|
||||||
|
if repair {
|
||||||
|
if p.wait {
|
||||||
|
select {
|
||||||
|
case p.entered <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
<-r.Context().Done()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ref := 2
|
||||||
|
if strings.Contains(req.Messages[len(req.Messages)-1].Content, `"proposal_ref":3`) {
|
||||||
|
ref = 3
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
p.attempts[ref]++
|
||||||
|
attempt := p.attempts[ref]
|
||||||
|
invalid := p.invalidFirstRevision && attempt == 1
|
||||||
|
p.mu.Unlock()
|
||||||
|
if ref == 3 {
|
||||||
|
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id := 2
|
||||||
|
if invalid {
|
||||||
|
id = 99
|
||||||
|
}
|
||||||
|
content = fmt.Sprintf(`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second"}}}`, id)
|
||||||
|
}
|
||||||
|
if decimalReferences {
|
||||||
|
switch {
|
||||||
|
case repair:
|
||||||
|
content = strings.NewReplacer(`"proposal_ref":2`, `"proposal_ref":2e0`, `"id":2`, `"id":2.0`).Replace(content)
|
||||||
|
case !validation:
|
||||||
|
content = strings.Replace(content, `"id":2`, `"id":2.0`, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(content)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
fmt.Fprintf(w, `{"model":"fixture","choices":[{"message":{"content":%s}}]}`, raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessReportsDefectOnOriginatingRepair(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
budget int
|
||||||
|
outcome string
|
||||||
|
recoveries int
|
||||||
|
}{{1, "exhausted", 1}, {2, "applied", 2}} {
|
||||||
|
t.Run(tc.outcome, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, tc.budget, false, false)
|
||||||
|
provider.mu.Lock()
|
||||||
|
provider.invalidFirstRevision = true
|
||||||
|
provider.mu.Unlock()
|
||||||
|
if code, _, stderr := f.run(t, true); code != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
doc := f.recoveries(t)
|
||||||
|
if *doc.Summary.ProposalResponses != 1 || *doc.Summary.ProposalResponsesWithFiltering != 0 || len(doc.Recoveries) != tc.recoveries {
|
||||||
|
t.Fatalf("unexpected recoveries: %+v", doc)
|
||||||
|
}
|
||||||
|
first := doc.Recoveries[0]
|
||||||
|
want := []reporting.FilteredProposal{{ProposalIndex: 1, Reason: "invalid_target_segment_id", Count: 1}}
|
||||||
|
if first.Operation != "correction_repair" || !reflect.DeepEqual(first.FilteredProposals, want) || !reflect.DeepEqual(first.Reasons, []string{"proposals_filtered"}) {
|
||||||
|
t.Fatalf("first repair recovery: %+v", first)
|
||||||
|
}
|
||||||
|
if tc.recoveries == 2 && !reflect.DeepEqual(doc.Recoveries[1].Reasons, []string{"correction_revised"}) {
|
||||||
|
t.Fatalf("successful repair recovery: %+v", doc.Recoveries[1])
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil || !strings.Contains(string(report), fmt.Sprintf("| 1 | 1 | 1 | %d | %s |", tc.budget, tc.outcome)) {
|
||||||
|
t.Fatalf("repair outcome: %s %v", report, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func setupCorrectionFixture(t *testing.T, attempts int, replacement, wait bool) (processFixture, *correctionProvider) {
|
||||||
|
t.Helper()
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
f.setSegments(t, "hello world", "second text", "third text")
|
||||||
|
provider := &correctionProvider{attempts: map[int]int{}, entered: make(chan struct{}, 1), wait: wait}
|
||||||
|
server := httptest.NewServer(provider)
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
resources := fmt.Sprintf("profiles:\n - id: proposal\n endpoint: %s/v1\n model: fixture\n - id: validation\n endpoint: %s/v1\n model: fixture\n", server.URL, server.URL)
|
||||||
|
if replacement {
|
||||||
|
resources += "collections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\n"
|
||||||
|
if err := os.WriteFile(filepath.Join(f.dir, "repair.txt"), []byte("OVERRIDE {{ .RepairRequestJSON }}"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
text := strings.Replace(string(data), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", attempts), 1)
|
||||||
|
text += "chunking:\n target_sections: 1\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
|
if err := os.WriteFile(f.config, []byte(text), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return f, provider
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessCorrectionRepairConversation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
debug, replacement, decimalReferences bool
|
||||||
|
}{{"default prompt without debug", false, false, false}, {"replacement with debug and equivalent reference spellings", true, true, true}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 2, tc.replacement, false)
|
||||||
|
provider.mu.Lock()
|
||||||
|
provider.decimalReferences = tc.decimalReferences
|
||||||
|
provider.mu.Unlock()
|
||||||
|
args := []string{"--verbose"}
|
||||||
|
if tc.debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 || stdout != "" {
|
||||||
|
t.Fatalf("code=%d stdout=%s stderr=%s", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
output := f.readTranscript(t, f.output)
|
||||||
|
if len(output.Segments) != 3 || output.Segments[0].Text != "Hello world" || output.Segments[1].Text != "Second text" || output.Segments[2].Text != "third text" {
|
||||||
|
t.Fatalf("transcript: %+v", output)
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, fragment := range []string{"## Correction repair", "| 1 | 1 | 1 | 2 | applied |", "| 1 | 1 | 2 | 1 | unavailable |"} {
|
||||||
|
if !strings.Contains(string(report), fragment) {
|
||||||
|
t.Errorf("missing %q in %s", fragment, report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
doc := f.errors(t)
|
||||||
|
recoveries := f.recoveries(t)
|
||||||
|
if doc.Summary.LLMOperations == nil || *doc.Summary.LLMOperations != 10 || *doc.Summary.CorrectionsApplied != 2 || *doc.Summary.Recoveries != 1 || !reflect.DeepEqual(doc.Summary, recoveries.Summary) || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
|
t.Fatalf("counts: errors=%+v recoveries=%+v stderr=%s", doc.Summary, recoveries.Summary, stderr)
|
||||||
|
}
|
||||||
|
if len(doc.Errors) != 1 || len(doc.Errors[0].Failures) != 1 || doc.Errors[0].Failures[0].Operation != "correction_repair" || doc.Errors[0].Failures[0].Reason != "provider_failed" {
|
||||||
|
t.Fatalf("errors: %+v", doc)
|
||||||
|
}
|
||||||
|
provider.mu.Lock()
|
||||||
|
requests := append([]repairWireRequest(nil), provider.requests...)
|
||||||
|
provider.mu.Unlock()
|
||||||
|
var base []repairWireMessage
|
||||||
|
var branches [][]repairWireMessage
|
||||||
|
for _, req := range requests {
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["corrections"]; ok {
|
||||||
|
base = req.Messages
|
||||||
|
}
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
|
||||||
|
branches = append(branches, req.Messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(base) != 2 || len(branches) != 3 || len(branches[0]) != 4 || len(branches[1]) != 6 || len(branches[2]) != 4 {
|
||||||
|
t.Fatalf("request lengths base=%d branches=%v", len(base), branches)
|
||||||
|
}
|
||||||
|
for _, messages := range branches {
|
||||||
|
for i := range base {
|
||||||
|
if messages[i] != base[i] {
|
||||||
|
t.Fatalf("changed prefix: %+v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if messages[2].Role != "assistant" || !strings.Contains(messages[2].Content, `"corrections"`) {
|
||||||
|
t.Fatalf("missing initial response: %+v", messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if branches[1][3] != branches[0][3] || branches[1][4].Role != "assistant" || strings.Contains(branches[2][3].Content, `"proposal_ref":2`) {
|
||||||
|
t.Fatalf("branch history: %+v", branches)
|
||||||
|
}
|
||||||
|
if tc.replacement && !strings.Contains(branches[0][3].Content, "OVERRIDE") {
|
||||||
|
t.Fatal("replacement not used")
|
||||||
|
}
|
||||||
|
if tc.debug {
|
||||||
|
for _, activity := range []string{"correction repair turn", "validation", "provider attempts"} {
|
||||||
|
if !strings.Contains(stderr, activity) {
|
||||||
|
t.Fatalf("missing repair progress %q: %s", activity, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
|
||||||
|
if err != nil || len(paths) != 1 {
|
||||||
|
t.Fatalf("execution path: %v %v", paths, err)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(paths[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var execution struct {
|
||||||
|
Summary struct {
|
||||||
|
ProviderAttempts int `json:"provider_attempts"`
|
||||||
|
} `json:"summary"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &execution); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := execution.Summary.ProviderAttempts; got != len(requests) {
|
||||||
|
t.Fatalf("execution provider attempts = %d, captured requests = %d", got, len(requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRepairDisabledAndOutputSelections(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
reporting, debug bool
|
||||||
|
}{{"neither", false, false}, {"debug only", false, true}, {"report only", true, false}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 0, false, false)
|
||||||
|
if !tc.reporting {
|
||||||
|
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(file, "reporting:\n enabled: false")
|
||||||
|
file.Close()
|
||||||
|
}
|
||||||
|
var args []string
|
||||||
|
if tc.debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, _, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
provider.mu.Lock()
|
||||||
|
requests := append([]repairWireRequest(nil), provider.requests...)
|
||||||
|
provider.mu.Unlock()
|
||||||
|
for _, req := range requests {
|
||||||
|
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
|
||||||
|
t.Fatal("repair request with zero budget")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := os.Stat(filepath.Join(f.dir, "report.md"))
|
||||||
|
if tc.reporting != (err == nil) {
|
||||||
|
t.Fatalf("report selection: %v", err)
|
||||||
|
}
|
||||||
|
if tc.debug {
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) == 0 {
|
||||||
|
t.Fatal("debug ledger missing")
|
||||||
|
}
|
||||||
|
} else if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected debug dir: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessCancellationDuringCorrectionRepair(t *testing.T) {
|
||||||
|
f, provider := setupCorrectionFixture(t, 2, false, true)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output, "--debug"}
|
||||||
|
command := exec.CommandContext(ctx, os.Args[0], args...)
|
||||||
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1")
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
command.Stderr = &stderr
|
||||||
|
if err := command.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-provider.entered:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatalf("repair not reached: %v", ctx.Err())
|
||||||
|
}
|
||||||
|
if err := command.Process.Signal(os.Interrupt); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := command.Wait()
|
||||||
|
if ctx.Err() != nil || err == nil {
|
||||||
|
t.Fatalf("cancellation: %v ctx=%v stderr=%s", err, ctx.Err(), stderr.String())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("transcript published: %v", err)
|
||||||
|
}
|
||||||
|
doc := f.errors(t)
|
||||||
|
if doc.Status != reporting.StatusFailed || len(doc.Errors) == 0 {
|
||||||
|
t.Fatalf("failed report: %+v", doc)
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil || !strings.Contains(string(data), "| 1 | 1 | 1 | 1 | interrupted |") {
|
||||||
|
t.Fatalf("partial report %s %v", data, err)
|
||||||
|
}
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) == 0 {
|
||||||
|
t.Fatal("partial ledger missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRepairWithoutOrdinaryReporting(t *testing.T) {
|
||||||
|
for _, debug := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) {
|
||||||
|
f, _ := setupCorrectionFixture(t, 2, false, false)
|
||||||
|
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(file, "reporting:\n enabled: false")
|
||||||
|
file.Close()
|
||||||
|
args := []string{"--verbose"}
|
||||||
|
if debug {
|
||||||
|
args = append(args, "--debug")
|
||||||
|
}
|
||||||
|
code, _, stderr := f.run(t, true, args...)
|
||||||
|
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
|
t.Fatalf("code=%d stderr=%s", code, stderr)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(f.dir, "report.md")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected report: %v", err)
|
||||||
|
}
|
||||||
|
if debug {
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 3 {
|
||||||
|
t.Fatalf("ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,19 +2,20 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
os.Exit(run())
|
os.Exit(run(os.Args[1:], os.Stdout, os.Stderr))
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() int {
|
func run(args []string, stdout, stderr io.Writer) int {
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
return cli.Run(ctx, os.Args[1:], os.Stdout, os.Stderr)
|
return cli.Run(ctx, args, stdout, stderr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"os/signal"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -19,10 +21,10 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/cli"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
||||||
"gopkg.in/yaml.v3"
|
"go.yaml.in/yaml/v4"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProcessSubprocessHelper(t *testing.T) {
|
func TestProcessSubprocessHelper(t *testing.T) {
|
||||||
@@ -38,10 +40,7 @@ func TestProcessSubprocessHelper(t *testing.T) {
|
|||||||
}
|
}
|
||||||
syscall.Umask(int(mask))
|
syscall.Umask(int(mask))
|
||||||
}
|
}
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
os.Exit(run(os.Args[i+1:], os.Stdout, os.Stderr))
|
||||||
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
|
|
||||||
stop()
|
|
||||||
os.Exit(code)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
@@ -118,53 +117,61 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
|
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
|
_, _ = io.WriteString(w, `{"error":{"message":"provider unavailable"}}`)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
|
||||||
|
if p.mode == "malformed-response" {
|
||||||
|
content = `{"corrections": [`
|
||||||
|
}
|
||||||
if p.mode == "repair-filter-later-failure" && !validation {
|
if p.mode == "repair-filter-later-failure" && !validation {
|
||||||
if proposalCall == 1 {
|
if proposalCall == 1 {
|
||||||
content = `{"wrong":true}`
|
content = `{"wrong":true}`
|
||||||
} else {
|
} else {
|
||||||
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X"},{"id":1,"original_text":"hello world","corrected_text":"Hello world"}]}`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(p.mode, "span-") {
|
if strings.HasPrefix(p.mode, "span-") {
|
||||||
switch p.mode {
|
switch p.mode {
|
||||||
case "span-clean", "span-filter":
|
case "span-clean", "span-filter":
|
||||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"}]}`
|
||||||
if p.mode == "span-filter" {
|
if p.mode == "span-filter" {
|
||||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":""},{"id":2,"original_text":"I I","corrected_text":"I"},{"id":3,"original_text":"uh","corrected_text":""}]}`
|
||||||
}
|
}
|
||||||
case "span-spacing":
|
case "span-spacing":
|
||||||
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" "}]}`
|
||||||
case "span-negative":
|
case "span-negative":
|
||||||
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":""}]}`
|
||||||
|
case "span-ambiguous":
|
||||||
|
content = `{"corrections":[{"id":1,"original_text":"Mage Armor","corrected_text":"ward"}]}`
|
||||||
|
if _, repair := request.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; repair {
|
||||||
|
content = `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if validation {
|
if validation {
|
||||||
if p.mode == "span-clean" || p.mode == "span-filter" {
|
if p.mode == "span-clean" || p.mode == "span-filter" {
|
||||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":2,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"},{"correction_index":2,"approved":true,"reason":"ok"}]}`
|
||||||
} else {
|
} else {
|
||||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||||
}
|
}
|
||||||
if p.mode == "span-negative" && validationCall == 2 {
|
if p.mode == "span-negative" && validationCall == 2 {
|
||||||
content = `{"validations":[{"correction_index":1,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
|
content = `{"validations":[{"correction_index":1,"approved":false,"reason":"would reverse meaning"}]}`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(p.mode, "decision-") && !validation {
|
if strings.HasPrefix(p.mode, "decision-") && !validation {
|
||||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"}]}`
|
||||||
if p.mode == "decision-mixed" {
|
if p.mode == "decision-mixed" {
|
||||||
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
|
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the"},{"id":2,"original_text":"cat","corrected_text":"dog"},{"id":3,"original_text":"bad","corrected_text":"good"},{"id":4,"original_text":"slow","corrected_text":"fast"}]}`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if validation && !strings.HasPrefix(p.mode, "span-") {
|
if validation && !strings.HasPrefix(p.mode, "span-") {
|
||||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
content = `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||||
if p.mode == "invalid-decision-after-repair" {
|
if p.mode == "invalid-decision-after-repair" {
|
||||||
if validationCall == 1 {
|
if validationCall == 1 {
|
||||||
content = `{"wrong":true}`
|
content = `{"wrong":true}`
|
||||||
} else {
|
} else {
|
||||||
content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
content = `{"validations":[{"correction_index":9,"approved":true,"reason":"ok"}]}`
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -175,13 +182,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
|
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2)
|
||||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
|
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "reason": "controlled review"})
|
||||||
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
|
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 4 {
|
||||||
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
|
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "reason": "conflicting review"})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
|
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
|
||||||
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
|
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "reason": "unexpected review"})
|
||||||
}
|
}
|
||||||
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||||
content = string(raw)
|
content = string(raw)
|
||||||
@@ -191,27 +198,37 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read the supplied candidate JSON independently of the bundled instruction prose.
|
||||||
|
type fixtureValidationItem struct {
|
||||||
|
CorrectionIndex int `json:"correction_index"`
|
||||||
|
OriginalText string `json:"original_text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func fixtureValidationItems(content string) ([]fixtureValidationItem, bool) {
|
||||||
|
for offset := 0; offset < len(content); offset++ {
|
||||||
|
if content[offset] != '[' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var items []fixtureValidationItem
|
||||||
|
if err := json.Unmarshal([]byte(content[offset:]), &items); err == nil && len(items) > 0 && items[0].CorrectionIndex > 0 {
|
||||||
|
return items, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
func fixtureValidationIndices(messages []struct {
|
func fixtureValidationIndices(messages []struct {
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
}) ([]int, error) {
|
}) ([]int, error) {
|
||||||
for _, message := range messages {
|
for _, message := range messages {
|
||||||
const marker = "Corrections to validate:"
|
if items, ok := fixtureValidationItems(message.Content); ok {
|
||||||
position := strings.LastIndex(message.Content, marker)
|
|
||||||
if position < 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var items []struct {
|
|
||||||
CorrectionIndex int `json:"correction_index"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
|
|
||||||
return nil, fmt.Errorf("decode validation request items: %w", err)
|
|
||||||
}
|
|
||||||
indices := make([]int, len(items))
|
indices := make([]int, len(items))
|
||||||
for i, item := range items {
|
for i, item := range items {
|
||||||
indices[i] = item.CorrectionIndex
|
indices[i] = item.CorrectionIndex
|
||||||
}
|
}
|
||||||
return indices, nil
|
return indices, nil
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("validation request items not found")
|
return nil, fmt.Errorf("validation request items not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -221,7 +238,6 @@ type processFixture struct {
|
|||||||
dir string
|
dir string
|
||||||
config string
|
config string
|
||||||
transcript string
|
transcript string
|
||||||
glossary string
|
|
||||||
output string
|
output string
|
||||||
provider *fixtureProvider
|
provider *fixtureProvider
|
||||||
server *httptest.Server
|
server *httptest.Server
|
||||||
@@ -233,15 +249,10 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
|
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
|
||||||
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
||||||
t.Cleanup(server.Close)
|
t.Cleanup(server.Close)
|
||||||
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
for _, id := range []string{"proposal", "validation"} {
|
|
||||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n", id)
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
configPath := filepath.Join(dir, "config.yml")
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
repairs := 0
|
repairs := 0
|
||||||
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
||||||
@@ -250,7 +261,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
repairs = 1
|
repairs = 1
|
||||||
}
|
}
|
||||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
var stageList []string
|
||||||
|
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
|
||||||
|
if name = strings.TrimSpace(name); name != "" {
|
||||||
|
stageList = append(stageList, " - module: "+name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pipeline := " stages: []\n"
|
||||||
|
if len(stageList) != 0 {
|
||||||
|
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
|
||||||
|
}
|
||||||
|
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n correction_repair_attempts: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
|
||||||
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
}
|
}
|
||||||
@@ -261,17 +282,14 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
glossary := filepath.Join(dir, "glossary.yaml")
|
|
||||||
if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
return processFixture{dir: dir, config: configPath, transcript: transcript,
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary,
|
|
||||||
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
|
||||||
if includeOutput {
|
if includeOutput {
|
||||||
args = append(args, "--output", f.output)
|
args = append(args, "--output", f.output)
|
||||||
}
|
}
|
||||||
@@ -340,11 +358,11 @@ func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transc
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
transcript, err := schema.ParseTranscriptJSON(raw)
|
transcript, err := schema.ParseSourceTranscriptJSON(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return transcript
|
return transcript.Canonical()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {
|
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {
|
||||||
@@ -705,6 +723,37 @@ func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) {
|
|||||||
f.assertNoStructuralRepairs(t, 3)
|
f.assertNoStructuralRepairs(t, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessRejectsAmbiguousSpanAtTheSameBoundaryWithOrWithoutRepair(t *testing.T) {
|
||||||
|
for _, budget := range []int{0, 1} {
|
||||||
|
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "span-ambiguous", "[grammar]")
|
||||||
|
f.setSegments(t, "Mage Armor and Mage Armor")
|
||||||
|
raw, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := strings.Replace(string(raw), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", budget), 1)
|
||||||
|
cfg += "protected_terms: [spells]\nglossary_collections:\n spells:\n entries:\n - {id: armor, name: Mage Armor, category: spell}\n"
|
||||||
|
if err := os.WriteFile(f.config, []byte(cfg), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if code, _, stderr := f.run(t, true, "--debug"); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 1 || !ledgerHasDecision(t, ledger[0], "deterministic_validator_decisions", "original_text_presence", false) ||
|
||||||
|
ledgerHasDecision(t, ledger[0], "deterministic_validator_decisions", "protected_terms", true) {
|
||||||
|
t.Fatalf("unexpected evidence: %+v", ledger)
|
||||||
|
}
|
||||||
|
if text := f.readTranscript(t, f.output).Segments[0].Text; text != "Mage Armor and Mage Armor" {
|
||||||
|
t.Errorf("ambiguous proposal changed segment to %q", text)
|
||||||
|
}
|
||||||
|
f.assertNoStructuralRepairs(t, 1+budget)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) {
|
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) {
|
||||||
f := newProcessFixture(t, "span-filter", "[spoken_word]")
|
f := newProcessFixture(t, "span-filter", "[spoken_word]")
|
||||||
f.setSegments(t, "uh hello", "I I think", "uh")
|
f.setSegments(t, "uh hello", "I I think", "uh")
|
||||||
@@ -756,7 +805,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
|
|||||||
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true)
|
||||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
|
if code != 0 || stdout != "" || stderr != "" {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
@@ -788,7 +837,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
||||||
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
|
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
@@ -799,35 +848,82 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
transcriptDir := filepath.Join(f.dir, "transcripts")
|
||||||
|
reportDir := filepath.Join(f.dir, "reports")
|
||||||
|
debugDir := filepath.Join(f.dir, "debug")
|
||||||
|
for _, dir := range []string{transcriptDir, reportDir} {
|
||||||
|
if err := os.Mkdir(dir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.output = filepath.Join(transcriptDir, "corrected.json")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
|
||||||
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||||
|
t.Fatalf("transcript: %s %v", output, err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(reportDir, name)); err != nil {
|
||||||
|
t.Fatalf("report %s: %v", name, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(transcriptDir, name)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("report %s appeared beside transcript: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dirs, err := filepath.Glob(filepath.Join(debugDir, "run-*"))
|
||||||
|
if err != nil || len(dirs) != 1 {
|
||||||
|
t.Fatalf("debug runs: %v %v", dirs, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dirs[0], "correction-ledger.json")); err != nil {
|
||||||
|
t.Fatalf("debug capture: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||||
var expectedOutput, expectedSummary string
|
var expectedOutput, expectedSummary string
|
||||||
for _, selection := range []struct{ reporting, debug bool }{
|
for _, selection := range []struct{ reporting, debug bool }{
|
||||||
{false, false}, {false, true}, {true, false}, {true, true},
|
{false, false}, {false, true}, {true, false}, {true, true},
|
||||||
} {
|
} {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
|
|
||||||
|
reportDir := filepath.Join(f.dir, "missing")
|
||||||
|
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
|
||||||
if selection.reporting {
|
if selection.reporting {
|
||||||
extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")}
|
reportDir = filepath.Join(f.dir, "selected-reports")
|
||||||
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
|
||||||
}
|
}
|
||||||
if selection.debug {
|
if selection.debug {
|
||||||
extra = append(extra, "--debug")
|
extra = append(extra, "--debug")
|
||||||
}
|
}
|
||||||
code, stdout, stderr := f.run(t, true, extra...)
|
code, stdout, stderr := f.run(t, true, extra...)
|
||||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
|
||||||
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
summary := strings.SplitN(stderr, "\n", 2)[0]
|
summary := ""
|
||||||
|
for _, line := range strings.Split(stderr, "\n") {
|
||||||
|
if strings.HasPrefix(line, "Completed:") {
|
||||||
|
summary = line
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
if expectedOutput == "" {
|
if expectedOutput == "" {
|
||||||
expectedOutput, expectedSummary = string(output), summary
|
expectedOutput, expectedSummary = string(output), summary
|
||||||
} else if string(output) != expectedOutput || summary != expectedSummary {
|
} else if string(output) != expectedOutput || summary != expectedSummary {
|
||||||
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
_, err := os.Stat(filepath.Join(f.dir, name))
|
_, err := os.Stat(filepath.Join(reportDir, name))
|
||||||
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
||||||
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
||||||
}
|
}
|
||||||
@@ -842,6 +938,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -855,7 +952,7 @@ func boolInt(value bool) int {
|
|||||||
|
|
||||||
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
||||||
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -867,14 +964,43 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
|||||||
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
|
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
|
||||||
t.Fatalf("model failure report: %+v", document)
|
t.Fatalf("model failure report: %+v", document)
|
||||||
}
|
}
|
||||||
if strings.Contains(stderr, f.provider.secret) {
|
}
|
||||||
t.Fatal("secret leaked in stderr")
|
|
||||||
|
func TestSubprocessCapturesMalformedProviderResponse(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "malformed-response", "[grammar]")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--debug")
|
||||||
|
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
|
||||||
|
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
|
||||||
|
}
|
||||||
|
responseCaptured := false
|
||||||
|
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry.IsDir() || !strings.HasSuffix(entry.Name(), "-response-payload.json") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var response struct {
|
||||||
|
FinalContent string `json:"final_content"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &response); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
responseCaptured = responseCaptured || response.FinalContent == `{"corrections": [`
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil || !responseCaptured {
|
||||||
|
t.Fatalf("missing malformed provider response: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
||||||
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
|
||||||
code, stdout, stderr := f.run(t, true)
|
code, stdout, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
||||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
}
|
}
|
||||||
@@ -890,7 +1016,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
||||||
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
||||||
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
|
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
|
||||||
}
|
}
|
||||||
@@ -920,7 +1046,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
|
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
|
||||||
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
|
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
|
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
@@ -938,7 +1064,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
|
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
|
||||||
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer stop()
|
defer stop()
|
||||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
|
||||||
command := exec.CommandContext(ctx, os.Args[0], args...)
|
command := exec.CommandContext(ctx, os.Args[0], args...)
|
||||||
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
@@ -955,8 +1081,9 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
err := command.Wait()
|
err := command.Wait()
|
||||||
if ctx.Err() != nil || err == nil {
|
var exit *exec.ExitError
|
||||||
t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String())
|
if ctx.Err() != nil || !errors.As(err, &exit) || exit.ExitCode() != 1 {
|
||||||
|
t.Fatalf("interrupt did not reach the failure exit: %v %v stderr=%q", err, ctx.Err(), stderr.String())
|
||||||
}
|
}
|
||||||
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
|
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
|
||||||
t.Fatalf("cancellation report: %+v", document)
|
t.Fatalf("cancellation report: %+v", document)
|
||||||
@@ -968,7 +1095,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
|
|||||||
|
|
||||||
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
||||||
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
||||||
code, _, stderr := f.run(t, true)
|
code, _, stderr := f.run(t, true, "--verbose")
|
||||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
@@ -982,32 +1109,63 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
code, _, _ := f.run(t, true)
|
code, _, _ := f.run(t, true)
|
||||||
if code != 1 || f.provider.callCount() != 0 {
|
if code != 2 || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||||
}
|
}
|
||||||
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
|
||||||
t.Fatalf("failed report: %+v", document)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
t.Fatalf("failed transcript published: %v", err)
|
t.Fatalf("failed transcript published: %v", err)
|
||||||
}
|
}
|
||||||
var recoveries reporting.RecoveriesDocument
|
}
|
||||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
|
|
||||||
t.Fatal(err)
|
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
|
||||||
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
|
||||||
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if recoveries.Status != reporting.StatusFailed {
|
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
|
||||||
t.Fatalf("recoveries status: %+v", recoveries)
|
if updated == string(original) {
|
||||||
|
t.Fatal("process fixture backend configuration was not found")
|
||||||
}
|
}
|
||||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
|
||||||
t.Fatalf("Markdown status: %s %v", raw, err)
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true)
|
||||||
|
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportDir := filepath.Join(f.dir, "failed-reports")
|
||||||
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, _, _ := f.run(t, true, "--report-dir", reportDir)
|
||||||
|
if code != 2 || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||||
|
}
|
||||||
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("report created before preflight: %s %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("failed report appeared in default directory: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("failed transcript published: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1026,7 +1184,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
|||||||
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
||||||
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
|
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
|
||||||
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(f.output); err != nil {
|
if _, err := os.Stat(f.output); err != nil {
|
||||||
@@ -1041,11 +1199,63 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
|
|||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||||
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
|
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
|
||||||
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSubprocessInfeasibleSectionTargetIsAnInputFailure(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--target-sections", "10")
|
||||||
|
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "target_sections") {
|
||||||
|
t.Fatalf("infeasible target: code=%d stdout=%q stderr=%q calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected transcript output: %v", err)
|
||||||
|
}
|
||||||
|
if document := f.errors(t); document.Status != reporting.StatusFailed {
|
||||||
|
t.Fatalf("report status: %+v", document)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessMissingCredentialIdentifiesRequiredEnvironmentVariable(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "success", "[grammar]")
|
||||||
|
f.provider.secret = ""
|
||||||
|
resources := filepath.Join(f.dir, "prompts.yml")
|
||||||
|
body, err := os.ReadFile(resources)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = bytes.Replace(body, []byte(" - id: proposal\n"), []byte(" - id: proposal\n api_key_required: true\n api_key_env: AUDITA_TEST_PROVIDER_KEY\n"), 1)
|
||||||
|
if err := os.WriteFile(resources, body, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
code, stdout, stderr := f.run(t, true)
|
||||||
|
if code != 1 || stdout != "" || f.provider.callCount() != 0 ||
|
||||||
|
!strings.Contains(stderr, "AUDITA_TEST_PROVIDER_KEY") || !strings.Contains(stderr, `proposal profile "proposal"`) {
|
||||||
|
t.Fatalf("missing credential: status=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
|
}
|
||||||
|
document := f.errors(t)
|
||||||
|
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 ||
|
||||||
|
document.Errors[0].Reason != "setup_failure" || strings.Contains(document.Errors[0].Message, "AUDITA_TEST_PROVIDER_KEY") {
|
||||||
|
t.Fatalf("expected compact setup failure in ordinary report: %+v", document)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessUnknownProfileFailsBeforeModelDispatchAndOutput(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--proposal-profile", "unknown")
|
||||||
|
if code != 2 || stdout != "" || f.provider.callCount() != 0 ||
|
||||||
|
!strings.Contains(stderr, "stage 1 (grammar) proposal profile") || !strings.Contains(stderr, "unknown") {
|
||||||
|
t.Fatalf("readiness: code=%d stdout=%q stderr=%q calls=%d", code, stdout, stderr, f.provider.callCount())
|
||||||
|
}
|
||||||
|
for _, path := range []string{f.output, filepath.Join(f.dir, "errors.yml")} {
|
||||||
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unexpected output at %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) {
|
func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) {
|
||||||
for _, mask := range []string{"0077", "0022"} {
|
for _, mask := range []string{"0077", "0022"} {
|
||||||
t.Run(mask, func(t *testing.T) {
|
t.Run(mask, func(t *testing.T) {
|
||||||
@@ -1093,47 +1303,33 @@ func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
func TestSubprocessAnnouncesActualOutputPaths(t *testing.T) {
|
||||||
for _, fail := range []bool{false, true} {
|
for _, fail := range []bool{false, true} {
|
||||||
t.Run(fmt.Sprint(fail), func(t *testing.T) {
|
t.Run(fmt.Sprint(fail), func(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[]")
|
f := newProcessFixture(t, "normal", "[]")
|
||||||
selected, unused := "selected-path-secret", "unused-path-secret"
|
reportDir := filepath.Join(f.dir, "reports")
|
||||||
f.provider.secret = selected
|
|
||||||
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
|
|
||||||
cfg, err := os.ReadFile(f.config)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
|
|
||||||
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
reportDir := filepath.Join(f.dir, unused)
|
|
||||||
if err := os.Mkdir(reportDir, 0700); err != nil {
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f.output = filepath.Join(reportDir, "out.json")
|
f.output = filepath.Join(reportDir, "out.json")
|
||||||
debugParent := filepath.Join(f.dir, selected)
|
debugParent := filepath.Join(f.dir, "diagnostics")
|
||||||
if fail {
|
if fail {
|
||||||
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
|
||||||
if (!fail && code != 0) || (fail && code != 1) {
|
if (!fail && code != 0) || (fail && code != 2) {
|
||||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||||
}
|
}
|
||||||
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
|
if !fail && (!strings.Contains(stderr, filepath.Join(reportDir, "report.md")) || !strings.Contains(stderr, debugParent)) {
|
||||||
t.Fatalf("credential in stderr: %q", stderr)
|
t.Fatalf("missing destination announcement: %q", stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr, "[REDACTED]") {
|
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
|
||||||
t.Fatalf("missing redacted announcement: %q", stderr)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
dirs, err := os.ReadDir(debugParent)
|
dirs, err := os.ReadDir(debugParent)
|
||||||
if err != nil || len(dirs) != 1 {
|
if !fail && (err != nil || len(dirs) != 1) {
|
||||||
t.Fatalf("debug location: %v %v", dirs, err)
|
t.Fatalf("debug location: %v %v", dirs, err)
|
||||||
}
|
}
|
||||||
if !fail {
|
if !fail {
|
||||||
@@ -1147,7 +1343,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
|||||||
|
|
||||||
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
||||||
f := newProcessFixture(t, "normal", "[grammar]")
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
code, _, baselineSummary := f.run(t, true)
|
code, _, baselineSummary := f.run(t, true, "--verbose")
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatal(baselineSummary)
|
t.Fatal(baselineSummary)
|
||||||
}
|
}
|
||||||
@@ -1162,16 +1358,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
|||||||
t.Errorf("debug directory: %v %v", dirs, err)
|
t.Errorf("debug directory: %v %v", dirs, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
|
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
|
||||||
t.Error(err)
|
t.Error(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.provider.mu.Unlock()
|
f.provider.mu.Unlock()
|
||||||
code, _, stderr := f.run(t, true, "--debug")
|
code, _, stderr := f.run(t, true, "--debug")
|
||||||
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
|
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
|
||||||
t.Fatalf("changed outcome: %d %q", code, stderr)
|
t.Fatalf("changed outcome: %d %q", code, stderr)
|
||||||
}
|
}
|
||||||
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
|
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
|
||||||
t.Fatalf("missing debug failures: %q", stderr)
|
t.Fatalf("missing debug failures: %q", stderr)
|
||||||
}
|
}
|
||||||
output, err := os.ReadFile(f.output)
|
output, err := os.ReadFile(f.output)
|
||||||
@@ -1204,7 +1400,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if invalid {
|
if invalid {
|
||||||
if code != 1 || !bytes.Equal(before, after) {
|
if code != 2 || !bytes.Equal(before, after) {
|
||||||
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
||||||
}
|
}
|
||||||
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
||||||
@@ -1218,15 +1414,18 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
f := newProcessFixture(t, "success", "[]")
|
f := newProcessFixture(t, "success", "[]")
|
||||||
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
||||||
// still an input and must not be overwritten during report initialization.
|
// still an input and must not be overwritten during report initialization.
|
||||||
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
|
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
|
||||||
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
|
||||||
|
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
before, err := os.ReadFile(loaded)
|
before, err := os.ReadFile(loaded)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
|
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
|
||||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||||
if code != 0 {
|
if code != 0 {
|
||||||
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
||||||
@@ -1240,7 +1439,203 @@ func TestSubprocessProfileSources(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
||||||
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
parentDir := filepath.Join(f.dir, "prompts", "catalog")
|
||||||
|
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
|
||||||
|
for _, directory := range []string{parentDir, childDir} {
|
||||||
|
if err := os.MkdirAll(directory, 0700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
|
||||||
|
child := filepath.Join(childDir, "override.yaml")
|
||||||
|
resource := filepath.Join(parentDir, "report.md")
|
||||||
|
for path, content := range map[string]string{
|
||||||
|
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
|
||||||
|
child: "prompts:\n - id: child\n extends: {id: source}\n",
|
||||||
|
resource: "Captured operator message.\n",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resourceBefore, err := os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f.output = resource
|
||||||
|
code, _, stderr = f.run(t, true, "--reporting=false")
|
||||||
|
resourceAfter, err := os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
f.output = filepath.Join(f.dir, "output.json")
|
||||||
|
code, _, stderr = f.run(t, true, "--report-dir", parentDir)
|
||||||
|
resourceAfter, err = os.ReadFile(resource)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
|
||||||
|
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessProgressReportingModes(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
visible bool
|
||||||
|
file bool
|
||||||
|
}{
|
||||||
|
{name: "ordinary"},
|
||||||
|
{name: "explicit", flags: []string{"--progress"}, visible: true},
|
||||||
|
{name: "debug", flags: []string{"--debug"}, visible: true},
|
||||||
|
{name: "debug with progress off", flags: []string{"--debug", "--progress=false"}},
|
||||||
|
{name: "quiet", flags: []string{"--quiet", "--progress"}, file: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
code, stdout, stderr := f.run(t, tc.file, tc.flags...)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, stderr)
|
||||||
|
}
|
||||||
|
for _, text := range []string{"Stage 1: grammar", "chunk 1", "proposal", "validation", "provider attempts"} {
|
||||||
|
if strings.Contains(stderr, text) != tc.visible {
|
||||||
|
t.Fatalf("progress fragment %q, expected visibility %v: %s", text, tc.visible, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.file {
|
||||||
|
if stdout != "" {
|
||||||
|
t.Fatalf("unexpected stdout with file output: %q", stdout)
|
||||||
|
}
|
||||||
|
} else if !json.Valid([]byte(stdout)) || !strings.Contains(stdout, "Hello world") {
|
||||||
|
t.Fatalf("stdout is not the transcript: %q", stdout)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessEmitsProgressBeforeProviderReturns(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "wait-cancel", "[grammar]")
|
||||||
|
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer stop()
|
||||||
|
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessSubprocessHelper$", "--",
|
||||||
|
"process", f.transcript, "--config", f.config, "--output", f.output, "--progress")
|
||||||
|
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||||
|
stderr, err := cmd.StderrPipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
stop()
|
||||||
|
_ = cmd.Wait()
|
||||||
|
}()
|
||||||
|
progress := make(chan string, 1)
|
||||||
|
go func() {
|
||||||
|
scanner := bufio.NewScanner(stderr)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if strings.Contains(line, "proposal") && strings.Contains(line, "chunk 1") {
|
||||||
|
select {
|
||||||
|
case progress <- line:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-f.provider.entered:
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("provider was not reached")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case line := <-progress:
|
||||||
|
if !strings.Contains(line, "grammar") || !strings.Contains(line, "started") {
|
||||||
|
t.Fatalf("progress omitted the running operation: %q", line)
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("proposal progress did not reach stderr while the provider was blocked")
|
||||||
|
}
|
||||||
|
if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err = cmd.Wait()
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if !errors.As(err, &exit) || exit.ExitCode() != 1 || ctx.Err() != nil {
|
||||||
|
t.Fatalf("termination did not reach the failure exit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type unavailableDiagnostics struct{}
|
||||||
|
|
||||||
|
func (unavailableDiagnostics) Write([]byte) (int, error) {
|
||||||
|
return 0, errors.New("progress destination failed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProgressDeliveryFailureKeepsCompletedProcessing(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
|
||||||
|
code := cli.Run(context.Background(), []string{"process", f.transcript, "--config", f.config, "--output", f.output, "--progress"}, io.Discard, unavailableDiagnostics{})
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("expected CommandKit presentation failure, got %d", code)
|
||||||
|
}
|
||||||
|
output, err := os.ReadFile(f.output)
|
||||||
|
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||||
|
t.Fatalf("transcript did not finish: %s, %v", output, err)
|
||||||
|
}
|
||||||
|
if doc := f.errors(t); doc.Status != reporting.StatusCompleted {
|
||||||
|
t.Fatalf("presentation changed report status: %+v", doc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type delayedProposalProgress struct {
|
||||||
|
delay time.Duration
|
||||||
|
delayed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *delayedProposalProgress) Write(p []byte) (int, error) {
|
||||||
|
if !w.delayed && bytes.Contains(p, []byte("proposal started")) {
|
||||||
|
w.delayed = true
|
||||||
|
time.Sleep(w.delay)
|
||||||
|
}
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProgressDeliveryDoesNotSpendModelDeadline(t *testing.T) {
|
||||||
|
const timeout = 500 * time.Millisecond
|
||||||
|
for _, showProgress := range []bool{false, true} {
|
||||||
|
f := newProcessFixture(t, "normal", "[grammar]")
|
||||||
|
t.Setenv("AUDITA_TEST_PROVIDER_KEY", f.provider.secret)
|
||||||
|
config, err := os.ReadFile(f.config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
config = bytes.Replace(config, []byte("llm:\n"), []byte("llm:\n operation_timeout: "+timeout.String()+"\n"), 1)
|
||||||
|
if err := os.WriteFile(f.config, config, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
args := []string{"process", f.transcript, "--config", f.config, "--output", f.output}
|
||||||
|
diagnostics := &delayedProposalProgress{delay: timeout + 250*time.Millisecond}
|
||||||
|
if showProgress {
|
||||||
|
args = append(args, "--progress")
|
||||||
|
}
|
||||||
|
code := cli.Run(context.Background(), args, io.Discard, diagnostics)
|
||||||
|
output, err := os.ReadFile(f.output)
|
||||||
|
if code != 0 || err != nil || !bytes.Contains(output, []byte("Hello world")) || f.provider.callCount() != 3 {
|
||||||
|
t.Fatalf("progress=%v, exit=%d, requests=%d, output=%q, read error=%v", showProgress, code, f.provider.callCount(), output, err)
|
||||||
|
}
|
||||||
|
if diagnostics.delayed != showProgress {
|
||||||
|
t.Fatalf("progress=%v, delayed=%v", showProgress, diagnostics.delayed)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
45
docs/adr/0001-releasekit.md
Normal file
45
docs/adr/0001-releasekit.md
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
# ADR 0001: Adopt ReleaseKit for release management
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted.
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-10-07
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Audita maintains custom release scripts, build tooling, delivery configuration,
|
||||||
|
and tests for mechanics shared across applications in the same suite.
|
||||||
|
[ReleaseKit](https://gitea.maximumdirect.net/eric/ReleaseKit) provides a common implementation and standard workflow for
|
||||||
|
those responsibilities.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Adopt ReleaseKit for shared release validation, source publication, binary
|
||||||
|
builds, version information, and hosted delivery. Audita will follow its standard
|
||||||
|
workflow and supply product identity, delivery targets, release notes, and
|
||||||
|
necessary product-specific checks.
|
||||||
|
|
||||||
|
Better maintainability and less locally owned release-focused code outweigh any
|
||||||
|
small advantage from retaining Audita's custom release procedure. Changes to
|
||||||
|
that procedure are an accepted consequence of standardization.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
Retaining the custom procedure would preserve its exact behavior and local
|
||||||
|
control. It would also retain responsibility for duplicated mechanics, tests,
|
||||||
|
and maintenance across projects. The shared implementation offers greater
|
||||||
|
long-term value.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Audita's release integration becomes smaller, with common mechanics and their
|
||||||
|
tests maintained upstream. Audita benefits from shared fixes and takes on
|
||||||
|
ReleaseKit dependency maintenance.
|
||||||
|
|
||||||
|
Adoption requires updates to release tooling, CI wiring, documentation, and
|
||||||
|
runtime version reporting. The [release policy](../policy/release.md) continues
|
||||||
|
to govern compatibility and release completion; the
|
||||||
|
[release procedure](../release.md) will describe the adopted workflow.
|
||||||
42
docs/adr/0002-glossary-configuration-and-preflight.md
Normal file
42
docs/adr/0002-glossary-configuration-and-preflight.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# ADR 0002: Glossary configuration and processing readiness
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
**Date:** 2026-11-10
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Operators need to reuse glossary collections across campaigns and select different
|
||||||
|
context and protected terms for each ordered correction. Configuration inspection
|
||||||
|
must remain useful without a transcript, model credentials, or a populated glossary.
|
||||||
|
A processing run must reject an unusable later correction before publishing outputs
|
||||||
|
or spending resources on earlier corrections.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
ConfigKit composes typed, named glossary collections with the application
|
||||||
|
configuration. Each pipeline entry selects context and protection independently;
|
||||||
|
Audita derives consumer views without changing the resolved configuration. For a
|
||||||
|
glossary correction, active recognized spellings are exempt from that entry's
|
||||||
|
protection view. Other protected spellings and validators remain effective.
|
||||||
|
|
||||||
|
After configuration resolution, Audita preflight checks the whole pipeline and
|
||||||
|
coordinates output preparation. It consumes the typed configuration and acquired
|
||||||
|
inputs, without reloading configuration or calling a provider. The output session
|
||||||
|
owns preparation, cleanup, and publication. See the [configuration](../config.md)
|
||||||
|
and [operations](../operations.md) contracts for current behavior.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
A separately parsed glossary per invocation would duplicate composition and make
|
||||||
|
per-entry selection and provenance harder to express. Requiring nonempty
|
||||||
|
collections in configuration validation would prevent inspection and reuse of
|
||||||
|
valid empty definitions. Preparing outputs or checking each entry only when it
|
||||||
|
runs could leave artifacts or incur model work before finding a later error.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
Configuration and runtime views have distinct responsibilities. Inspection can
|
||||||
|
show configured selections and origins without presenting derived vocabulary as
|
||||||
|
configuration. Preflight and the output session need a clear handoff, and the
|
||||||
|
processing pipeline must carry each entry's own effective settings and identity.
|
||||||
59
docs/adr/0003-module-focus-and-editorial-acceptance.md
Normal file
59
docs/adr/0003-module-focus-and-editorial-acceptance.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# ADR 0003: Module focus and editorial acceptance
|
||||||
|
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
**Date:** 2026-11-11
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Audita gives each correction module a focused task. A prompt review suggested
|
||||||
|
making that task's scope an acceptance condition, because validators can accept
|
||||||
|
minor editorial changes that the proposing module was not asked to make.
|
||||||
|
|
||||||
|
The maintainer's practical testing established why this difference is deliberate.
|
||||||
|
Models persistently propose minor editorial cleanup when it is clearly warranted
|
||||||
|
in the supplied text. Instructions strict enough to suppress that cleanup also
|
||||||
|
suppress desirable edits. Enforcing exclusive module scopes would therefore
|
||||||
|
trade away useful corrections to achieve a separation that does not serve the
|
||||||
|
application's editorial goals.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Module scope guides the corrections Audita asks a model to propose. It is not an
|
||||||
|
exhaustive boundary on acceptable edits. Minor editorial cleanup is permitted
|
||||||
|
across modules and their validators even when the proposal prompt does not
|
||||||
|
request it.
|
||||||
|
|
||||||
|
Validators must not reject an otherwise acceptable minor editorial change solely
|
||||||
|
because it falls outside the originating module's requested focus. Existing
|
||||||
|
checks for applicability, protected terms, meaning, and other domain concerns
|
||||||
|
continue to apply. This allowance does not guarantee that any particular edit
|
||||||
|
passes validation or application, nor does it authorize arbitrary rewriting.
|
||||||
|
|
||||||
|
Proposal, validation, and repair instructions may differ to serve their roles.
|
||||||
|
Their consistency should be assessed against this distinction, rather than by
|
||||||
|
requiring identical lists of allowed edits. Repair feedback should not turn
|
||||||
|
module focus into an exclusive acceptance rule.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
**Strict module-aware acceptance**, whether implemented in shared validators,
|
||||||
|
separate review definitions, or an additional scope check, would make stage
|
||||||
|
boundaries easier to describe. It is rejected because the maintainer's testing
|
||||||
|
shows that suppressing incidental cleanup also suppresses desirable work.
|
||||||
|
|
||||||
|
**Request every kind of cleanup in every module** would align proposal requests
|
||||||
|
with the broader acceptance policy, but would dilute the focused tasks. Permitting
|
||||||
|
incidental cleanup does not require asking each stage to perform every task.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
An edit accepted outside a module's requested focus is not, by itself, a design
|
||||||
|
defect. Prompt maintenance and evaluation should preserve useful corrections
|
||||||
|
while checking substantive editorial quality. They should not introduce strict
|
||||||
|
scope enforcement as an apparent consistency fix.
|
||||||
|
|
||||||
|
The [architecture policy](../policy/architecture.md#correction-and-concurrency-invariants)
|
||||||
|
records the boundary. Reconsidering this accepted decision requires new evidence
|
||||||
|
and explicit supersession under the ADR lifecycle, rather than rediscovering the
|
||||||
|
same prompt/validator difference during each review.
|
||||||
@@ -1,6 +1,15 @@
|
|||||||
# Architectural Decision Records
|
# Architectural Decision Records
|
||||||
|
|
||||||
No ADRs have been recorded here. Use the
|
## Accepted
|
||||||
|
|
||||||
|
- [ADR 0001: Adopt ReleaseKit for release management](0001-releasekit.md) —
|
||||||
|
accepted 2026-10-07.
|
||||||
|
- [ADR 0002: Glossary configuration and processing readiness](0002-glossary-configuration-and-preflight.md) —
|
||||||
|
accepted 2026-11-10.
|
||||||
|
- [ADR 0003: Module focus and editorial acceptance](0003-module-focus-and-editorial-acceptance.md) —
|
||||||
|
accepted 2026-11-11.
|
||||||
|
|
||||||
|
Use the
|
||||||
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)
|
||||||
for significant decisions. Current architecture belongs in the
|
for significant decisions. Current architecture belongs in the
|
||||||
[architecture policy](../policy/architecture.md); implementation status belongs in
|
[architecture policy](../policy/architecture.md); implementation status belongs in
|
||||||
|
|||||||
124
docs/cli.md
124
docs/cli.md
@@ -1,95 +1,53 @@
|
|||||||
# Audita CLI Reference
|
# Audita CLI Reference
|
||||||
|
|
||||||
The shared version-2 configuration parser and offline profile inspection serve
|
Audita uses CommandKit for routing and the ConfigKit and PromptKit integrations
|
||||||
`process`, `config validate`, and `config print-effective`.
|
for their standard inspection and request commands. Run `audita --help` or
|
||||||
The [configuration reference](config.md) owns fields, defaults, and validation;
|
command-specific `--help` for the installed command inventory. The adopted
|
||||||
the [migration guide](migrations/v2.md) maps retired inputs.
|
[CommandKit consumer guide](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/consumers/guide.md)
|
||||||
|
describes the standard command grammar, inspection, comparison, preview,
|
||||||
|
validation, smoke tests, and their provider-specific caveats. Help and version
|
||||||
|
do not load application or model configuration.
|
||||||
|
|
||||||
## Commands and help
|
## Processing
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
audita process <transcript.json> --glossary <glossary.yml> [flags]
|
audita process [flags] transcript.json
|
||||||
audita config validate --config <config.yml> [flags]
|
|
||||||
audita config print-effective [flags]
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Root invocation, `--help`, `-h`, and command help succeed without loading a config
|
`--config` selects a main configuration and `--overlay` selects its overlay.
|
||||||
file or resolving secrets. Unknown commands and flags return exit code `2`.
|
`--output` writes the transcript to a file; omission selects stdout. The shared
|
||||||
`process` accepts exactly one transcript path, either before the flags or after
|
`--debug` flag enables retained debug artifacts, and `--quiet` suppresses optional
|
||||||
them. With flags first, put the transcript last. Both config commands reject
|
messages. Quiet processing requires an output file or `/dev/null`. See
|
||||||
positional arguments and process-only flags.
|
[operations](operations.md) for output and report lifecycle.
|
||||||
|
|
||||||
All three commands accept `--config <path>` and `--overlay <name>`. `config validate`
|
Progress for processing goes to stderr. Use `--progress` to show stage and
|
||||||
requires an explicit nonempty `--config`; the other commands select the explicit
|
chunk activity, model proposal, validation and repair operations, and observed
|
||||||
file, then `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`.
|
counts as work completes. Debug mode enables progress by default.
|
||||||
Missing system candidates are skipped; inaccessible selected files are errors.
|
`--progress=false` disables it, and quiet mode suppresses it even when
|
||||||
No file means a configuration error. `AUDITA_CONFIG` is ignored. The selected
|
`--progress` is explicit.
|
||||||
overlay is loaded beside the main file, not used to locate profile files.
|
|
||||||
|
|
||||||
## Setting overrides
|
Explicit process flags override root settings only. `--glossaries` and
|
||||||
|
`--protected-terms` each accept one JSON array of collection names or `null`;
|
||||||
|
`[]` clears the selection. Set pipeline order and per-entry overrides in YAML.
|
||||||
|
Other supplied flags cover profiles, output schema, description, reporting,
|
||||||
|
chunking, normalization, validation budget, and debug
|
||||||
|
work directory. `audita process --help` lists the available flags. An omitted flag
|
||||||
|
does not contribute an override. [Configuration](config.md) defines their values
|
||||||
|
and inheritance.
|
||||||
|
|
||||||
Only flags actually supplied override file or overlay settings. `--modules ''`
|
## Inspection and outcomes
|
||||||
clears the module pipeline, and an explicit numeric zero reaches configuration
|
|
||||||
validation. The shared setting flags are:
|
|
||||||
|
|
||||||
| Setting | Flags |
|
The application configuration scope uses ConfigKit's native effective values,
|
||||||
| --- | --- |
|
source evidence, explanation, comparison, and redaction. PromptKit supplies its
|
||||||
| Profile selection | `--proposal-profile`, `--validation-profile` |
|
own configuration and resource scopes and request preparation, validation, and
|
||||||
| Pipeline and output | `--modules`, `--output-schema` |
|
explicit smoke diagnostics. Inspection does not require a processing-ready
|
||||||
| Validation budget | `--validation-max-prompt-tokens` |
|
pipeline or contact a provider. Processing preflight separately checks the selected
|
||||||
| Chunking | `--target-sections`, `--max-section-tokens`, `--min-section-tokens` |
|
profiles and glossary entries. Smoke tests can contact providers and are run
|
||||||
| Normalization (numeric seconds) | `--normalize-max-segment-gap`, `--normalize-ellipsis-gap`, `--normalize-max-segment-duration`, `--normalize-max-segment-tokens` |
|
only on explicit request.
|
||||||
| Confidence thresholds | `--glossary-confidence-threshold`, `--homophones-confidence-threshold`, `--spoken-word-confidence-threshold`, `--grammar-confidence-threshold` |
|
|
||||||
| Context | `--transcript-description` |
|
|
||||||
| Reporting | `--reporting[=true\|false]`, `--report-dir DIR` |
|
|
||||||
| Debug | `--debug[=true\|false]`, `--work-dir DIR` |
|
|
||||||
|
|
||||||
`process` alone accepts `--glossary` (required) and `--output`.
|
Process stdout contains transcript JSON when no output file is selected. Ordinary
|
||||||
An omitted or empty output value selects stdout. Nonempty output filenames are
|
reports may record recoverable model failures even when processing completes.
|
||||||
used literally, including whitespace.
|
Exit status `0` means success, `1` runtime or output failure, `2` invalid usage
|
||||||
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
|
or input, and native comparison may return `3` for differences. A missing
|
||||||
reports; an explicit false overrides a true file or overlay setting. With file
|
processing profile or required glossary selection is an input failure. See the
|
||||||
output, reports go beside the transcript and `--report-dir` has no effect. With
|
[output contract](integrations/output-files.md) and [subprocess contract](integrations/subprocess.md).
|
||||||
stdout output, `--report-dir` selects the report directory, defaulting to the
|
|
||||||
invocation working directory. `--debug` separately enables retained detailed
|
|
||||||
artifacts under `--work-dir`. Configuration inspection does not create or probe
|
|
||||||
these output directories.
|
|
||||||
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
|
|
||||||
removed. Models and generation controls belong to PromptKit profiles, backend
|
|
||||||
credentials and capacity to configuration, and retry/deadline settings to the
|
|
||||||
shared LLM section. Legacy configuration environment overrides are ignored.
|
|
||||||
|
|
||||||
## Streams and exit status
|
|
||||||
|
|
||||||
`config validate` loads settings and both selected PromptKit profiles offline,
|
|
||||||
then prints `config is valid` on success. `config print-effective` accepts
|
|
||||||
`--view settings|profiles|explain` and defaults to `settings`. Settings writes
|
|
||||||
ConfigKit's redacted effective-settings YAML. Profiles writes an Audita-owned
|
|
||||||
YAML summary of selected ID, backend, model, sampling controls, per-attempt
|
|
||||||
timeout, service tier, and reasoning effort for each role. It omits credentials,
|
|
||||||
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
|
|
||||||
sequence of field paths, source origins, and recorded derivations without
|
|
||||||
resolved setting values. These views share the same settings and profile
|
|
||||||
validation path as `process`. Unknown views are usage errors. The effective
|
|
||||||
settings YAML is for inspection and is not guaranteed reloadable.
|
|
||||||
|
|
||||||
For `process`, `--output` writes the corrected transcript to a file and leaves
|
|
||||||
stdout empty on success. Without it, stdout carries transcript JSON only on
|
|
||||||
success. Enabled reporting creates `errors.yml`, `recoveries.yml`, and
|
|
||||||
`report.md` in the selected directory. Successful stderr begins with
|
|
||||||
`Completed: N corrections applied; E work units with errors; R recoveries.`
|
|
||||||
Report and debug locations follow only when created. Model-operation failures
|
|
||||||
can coexist with a completed run and applied corrections; read the YAML reports
|
|
||||||
for details. Debug artifacts are created only with `--debug`. See the
|
|
||||||
[report contract](integrations/output-files.md).
|
|
||||||
|
|
||||||
Exit codes are `0` for success, `1` for runtime or output failure, and `2` for
|
|
||||||
usage or configuration input errors. Invalid settings and profiles return `2`;
|
|
||||||
broken embedded assets, caller cancellation, fatal processing, and required
|
|
||||||
output failures return `1`. A late report-write failure can follow a published
|
|
||||||
transcript; exit status and report status remain authoritative.
|
|
||||||
Configuration commands inspect profiles without contacting a provider.
|
|
||||||
|
|
||||||
See the [input files](integrations/transcript-glossary-files.md),
|
|
||||||
[subprocess contract](integrations/subprocess.md), and
|
|
||||||
[artifact lifecycle](operations.md) for their respective details.
|
|
||||||
|
|||||||
174
docs/config.md
174
docs/config.md
@@ -1,88 +1,114 @@
|
|||||||
# Audita Configuration
|
# Audita Configuration
|
||||||
|
|
||||||
This reference describes the version-2 configuration loader in `internal/core/config`.
|
Audita uses version 3 application configuration. See the [migration guide](migrations/v3.md)
|
||||||
Use the maintained [local](../examples/minimal-config.yml) and
|
and [examples](../examples/) for complete projects. Command syntax is owned by
|
||||||
[service](../examples/production-config.yml) examples with their PromptKit
|
[the CLI reference](cli.md).
|
||||||
[profiles](../examples/profiles/) and optional [testing overlay](../examples/overlays/testing.yml).
|
|
||||||
Choose real models for your endpoint; a testing profile makes no cost or quality
|
|
||||||
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
|
|
||||||
|
|
||||||
## Loading and precedence
|
## Loading and composition
|
||||||
|
|
||||||
The application selects one main YAML file: an explicit `--config`, then
|
An explicit `--config` selects one main YAML file. Without it, ConfigKit searches
|
||||||
`/usr/local/etc/audita/config.yml` if present, then `/etc/audita/config.yml`
|
the working directory, user configuration directory, and Unix system directories
|
||||||
if present. Missing system candidates are skipped; an explicit missing or empty
|
for `audita` configuration. If no file is found, built-in defaults apply. An
|
||||||
path, inaccessible selected file, or absence of any selected file is an error.
|
explicit invalid selection or failure of a selected file is an error; search does
|
||||||
`config validate` requires an explicit file. ConfigKit loads built-in defaults,
|
not continue after a selected file fails. An omitted version uses the v3 schema;
|
||||||
the selected file, an optional named overlay from `overlays/<name>.yml` beside the
|
a supplied version must be `3`.
|
||||||
main file, and only explicitly supplied setting overrides, in that order. Imports
|
|
||||||
and secret-directory bootstrap are disabled. Unknown fields and missing, null,
|
|
||||||
or non-integer `version: 2` are errors. Version-1 files require migration; no
|
|
||||||
compatibility loader or implicit environment configuration remains.
|
|
||||||
|
|
||||||
Validation runs after overlays, explicit overrides, and derived settings have
|
Operator sources may use ConfigKit imports. The selected main file provides the
|
||||||
resolved. An override can therefore repair an invalid lower-precedence setting
|
base for an optional `overlays/<name>.yml`, optional `glossary.yml`, and recursive
|
||||||
when that setting can be decoded. ConfigKit retains field origins and derivations
|
`glossary/` YAML files. Without a main source, automatic companion files are not
|
||||||
for effective-configuration inspection. Its effective YAML redacts resolved
|
loaded from the working directory. ConfigKit composes sources, resolves native
|
||||||
credentials.
|
inheritance, applies explicit overrides, records derived defaults, and validates
|
||||||
|
the result. Its [configuration reference](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md)
|
||||||
|
owns native composition and provenance semantics.
|
||||||
|
|
||||||
Profile files are resolved against the selected main file's directory, or may
|
### PromptKit profiles and operator prompts
|
||||||
use an absolute `profiles_dir`. Both selected profiles must exist and select a
|
|
||||||
registered backend. Their effective endpoints must match that backend's endpoint
|
PromptKit loads its default catalog and Audita's embedded prompts, followed by
|
||||||
after surrounding whitespace and trailing slashes are removed. Profiles cannot
|
optional `prompts.yml` and recursive `prompts/` YAML files beside the application
|
||||||
supply a competing `api_key_env` lookup. The backend credential supplies the
|
main. Native imports, resource collections, inheritance, and replacement apply.
|
||||||
runtime key when required; inspection makes no provider call. The
|
Profiles, backends, models, credentials, admission, and provider parameters belong
|
||||||
[CLI inspection views](cli.md#streams-and-exit-status) show selected controls and
|
to PromptKit configuration; see its [format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/docs/formats.md).
|
||||||
field provenance separately from ConfigKit's redacted settings YAML.
|
The engine captures its sources for an invocation. Edit files and run a new
|
||||||
|
invocation to pick up changes. Inspection does not require model credentials.
|
||||||
|
|
||||||
|
## Pipeline and glossary selections
|
||||||
|
|
||||||
|
`glossary_collections` maps names to collections with an ordered `entries` list.
|
||||||
|
Each entry has an `id`, `name`, and `category`; `description`, `aliases`, and
|
||||||
|
`plural` provide optional context. Entries are identified within their collection
|
||||||
|
by `id`. Root `glossaries` and `protected_terms` select collection names. If root
|
||||||
|
protection is omitted, it derives from the root glossary selection.
|
||||||
|
|
||||||
|
`pipeline.stages` is an ordered list of `module` records; supported modules are
|
||||||
|
`glossary`, `homophones`, `spoken_word`, and `grammar`. The default sequence is
|
||||||
|
`glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`. Repeated modules
|
||||||
|
run separately; `[]` runs normalization without model processing. An optional
|
||||||
|
nonempty, unique `id` identifies a record in evidence. Each record may override
|
||||||
|
`glossaries`, `protected_terms`, and fields under `llm`,
|
||||||
|
`validation`, and `context`. Omitted fields inherit their resolved root values.
|
||||||
|
Explicit empty lists and null list selections clear a selection.
|
||||||
|
|
||||||
|
Each entry receives its selected context and independent protection. Glossary
|
||||||
|
correction exempts active recognized spellings from protection for that entry.
|
||||||
|
A longer exempt name still takes precedence over a shorter protected spelling
|
||||||
|
inside it. A glossary correction must have at least one selected entry at
|
||||||
|
processing preflight. Empty definitions and empty protection remain valid
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
The bundled prompts use English. The modules request complementary work:
|
||||||
|
|
||||||
|
- `glossary`: context-supported corrections toward selected glossary terms.
|
||||||
|
- `homophones`: plausible spoken-word mistranscriptions, including glossary terms.
|
||||||
|
- `spoken_word`: conservative filler, hesitation, and accidental repetition cleanup.
|
||||||
|
- `grammar`: punctuation, capitalization, spacing, and whole-word `a`/`an` cleanup.
|
||||||
|
|
||||||
|
Otherwise acceptable minor editorial cleanup may be accepted across modules, as
|
||||||
|
specified by the [editorial acceptance boundary](policy/architecture.md#correction-and-concurrency-invariants).
|
||||||
|
|
||||||
## Settings
|
## Settings
|
||||||
|
|
||||||
| Field | Default | Rule |
|
| Setting | Default | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `profiles_dir` | `profiles` | Nonempty; profile files are resolved relative to the selected main file. |
|
| `llm.proposal_profile` | unset | PromptKit profile for proposals; required for processing entries. |
|
||||||
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
|
| `llm.validation_profile` | effective proposal profile | Profile for model validation; null clears a selection before fallback. |
|
||||||
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
|
| `llm.correction_repair_profile` | effective proposal profile | Profile for correction repair; null clears an inherited selection before fallback. |
|
||||||
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
|
| `llm.correction_repair_attempts` | `1` | Nonnegative repair attempts per correction; zero disables correction repair. |
|
||||||
| `backends.<id>.credential` | Unset | Optional environment-variable secret reference; omission or null means unauthenticated use. |
|
| `llm.transient_retries` | `2` | Additional transient attempts per generation. |
|
||||||
| `llm.proposal_profile` | None | Required nonempty profile ID. |
|
| `llm.repair_attempts` | `2` | Structural repairs, from zero through three. |
|
||||||
| `llm.validation_profile` | Final proposal profile | Omission inherits the final proposal selection; explicit empty or null fails. |
|
| `llm.operation_timeout` | `15m` | Positive duration per logical operation. |
|
||||||
| `llm.transient_retries` | `2` | Nonnegative additional attempts per generation. |
|
|
||||||
| `llm.repair_attempts` | `2` | Additional structural repair generations, from zero through three. |
|
|
||||||
| `llm.operation_timeout` | `15m` | Positive Go duration string per logical call. |
|
|
||||||
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module keys; repeats are meaningful and an explicit empty list runs no correction modules. |
|
|
||||||
| `output.schema` | `bare-segments` | `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md). |
|
|
||||||
| `validation.max_prompt_tokens` | `2048` | Positive validation batching budget. |
|
| `validation.max_prompt_tokens` | `2048` | Positive validation batching budget. |
|
||||||
| `chunking.target_sections` | Unset | Optional positive target; omission or null selects automatic chunking. |
|
| `context.description` | empty | Prompt context; at most 500 bytes after trimming. |
|
||||||
| `chunking.max_section_tokens` | `8192` | Positive maximum estimate. |
|
| `chunking.target_sections` | unset | Optional positive count. Processing requires exactly this many nonempty sections when specified. An infeasible count is an input error. |
|
||||||
| `chunking.min_section_tokens` | `2048` | Positive and no greater than maximum. |
|
| `chunking.max_section_tokens`, `chunking.min_section_tokens` | `8192`, `2048` | The maximum bounds a section's estimated tokens, except for an oversized segment in a section of its own. The minimum is validated against the maximum and retained for compatibility. |
|
||||||
| `normalization.max_segment_gap` | `4.0` | Nonnegative numeric seconds. |
|
| `normalization.max_segment_gap`, `normalization.ellipsis_gap` | `4`, `3.5` | Nonnegative seconds. |
|
||||||
| `normalization.ellipsis_gap` | `3.5` | Nonnegative numeric seconds. |
|
| `normalization.max_segment_duration`, `normalization.max_segment_tokens` | `60`, `2048` | Positive seconds and token estimate. |
|
||||||
| `normalization.max_segment_duration` | `60.0` | Positive numeric seconds. |
|
| `output.schema` | `bare-segments` | Output format; see [output contracts](integrations/output-files.md). |
|
||||||
| `normalization.max_segment_tokens` | `2048` | Positive token estimate. |
|
| `reporting.enabled`, `reporting.stdout_dir` | `true`, `.` | Ordinary report capture and default directory for stdout output. |
|
||||||
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
|
| `diagnostics.work_dir` | `/tmp/audita` | Parent directory for optional debug capture. |
|
||||||
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
|
|
||||||
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
|
|
||||||
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; ignored for file output or disabled reporting. |
|
|
||||||
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
|
|
||||||
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
|
|
||||||
|
|
||||||
Module keys select correction modules; validator chains remain built into them.
|
For example, an entry can disable correction repair or select another profile:
|
||||||
The loader preserves module order and repetitions. Normalization time settings
|
|
||||||
accept numeric seconds only. The operation timeout uses a duration string.
|
|
||||||
`reporting.stdout_dir` must be nonblank only for reporting-enabled stdout runs;
|
|
||||||
`diagnostics.work_dir` must be nonblank only with debug enabled. These path
|
|
||||||
requirements are checked when choosing process destinations, not during
|
|
||||||
configuration inspection.
|
|
||||||
|
|
||||||
## Credentials
|
```yaml
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
correction_repair_attempts: 2
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: grammar
|
||||||
|
llm:
|
||||||
|
correction_repair_profile: review
|
||||||
|
- module: spoken_word
|
||||||
|
llm:
|
||||||
|
correction_repair_attempts: 0
|
||||||
|
```
|
||||||
|
|
||||||
A backend credential is a ConfigKit secret reference containing an environment
|
Processing profiles are selected explicitly; no profile or glossary is selected
|
||||||
variable name, such as `credential: AUDITA_SERVICE_KEY`. Every reference that
|
automatically. The validation selection falls back after root and per-entry
|
||||||
survives overlay and override resolution must name a set, nonempty variable,
|
settings are combined. An explicit root correction repair profile is inherited
|
||||||
including credentials on backends unused by the selected roles. A replacement
|
by each entry unless overridden or cleared with null; otherwise the entry uses
|
||||||
reference resolves only the replacement; null removes the reference. General
|
its effective proposal profile. The repair profile is checked only when its
|
||||||
`AUDITA_*` and `OPENROUTER_API_KEY` variables have no implicit setting effect.
|
attempt budget is positive. Each correction repair attempt is a logical call
|
||||||
Secret values are available to the runtime through the resolved reference, while
|
that can also use the configured transient retries and structural repairs. Null
|
||||||
ConfigKit's effective output and provenance contain no resolved secret values.
|
does not disable validation. Numeric zero and false
|
||||||
|
retain their field meaning, subject to validation. Inspect recorded origins and
|
||||||
Current invocation syntax belongs in [CLI](cli.md).
|
default dependencies using the [native commands](cli.md).
|
||||||
|
|||||||
@@ -41,7 +41,9 @@ ownership. Commit discipline and agent Git restrictions are in the
|
|||||||
|
|
||||||
## Baseline Validation
|
## Baseline Validation
|
||||||
|
|
||||||
Use Go 1.25.5 or newer. From the repository root:
|
Use Go 1.27.0 or newer for source builds. The selected release toolchain is
|
||||||
|
documented in the [release procedure](release.md#prerequisites). From the
|
||||||
|
repository root:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go build -o ./bin/audita ./cmd/audita
|
go build -o ./bin/audita ./cmd/audita
|
||||||
|
|||||||
@@ -1,26 +1,26 @@
|
|||||||
# OpenAI-Compatible LLM Integration
|
# OpenAI-Compatible LLM Integration
|
||||||
|
|
||||||
Audita uses the pinned [PromptKit](https://gitea.maximumdirect.net/eric/promptkit)
|
Audita uses the pinned [PromptKit](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09)
|
||||||
built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing,
|
built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing,
|
||||||
provider envelope parsing, transient retries, per-generation timeouts, and JSON
|
provider envelope parsing, transient retries, per-generation timeouts, and JSON
|
||||||
Schema repair. A backend's configured base endpoint must support the selected
|
Schema repair. A backend's configured base endpoint must support the selected
|
||||||
profile's model and structured JSON-schema responses; local configuration
|
profile's model and structured JSON-schema responses; local configuration
|
||||||
inspection does not contact the provider.
|
inspection does not contact the provider.
|
||||||
|
|
||||||
Audita selects a role profile, supplies its registered backend credential, and
|
Audita selects role profiles independently for each pipeline entry and supplies
|
||||||
passes the embedded response schema as a complete per-call output contract. It
|
its domain response schema and per-operation budget. PromptKit owns backend and
|
||||||
prepares each call once, checks that the prepared target still matches the
|
credential selection, request preparation, transport, retries, repairs, and
|
||||||
inspected backend and endpoint, then executes that same prepared request. A
|
admission. Audita executes the same prepared handle and retains its own domain
|
||||||
profile cannot introduce a separate credential environment lookup. An absent
|
validation. Profiles may use native endpoint and credential settings; local
|
||||||
backend credential permits an unauthenticated local service.
|
inspection does not resolve a credential value or call the provider.
|
||||||
|
|
||||||
`llm.transient_retries` counts additional attempts for eligible transient
|
`llm.transient_retries` counts additional attempts for eligible transient
|
||||||
failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
failures; explicit zero makes one attempt. `llm.repair_attempts` counts
|
||||||
additional generations for schema-invalid model output. The operation timeout
|
additional generations for schema-invalid model output. The operation timeout
|
||||||
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
covers preparation, capacity waiting, retries, generation, and repair. Caller
|
||||||
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
|
||||||
for defaults and [PromptKit's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.2/docs/consumers/pkg-promptkit.md)
|
for defaults and the [PromptKit consumer contract](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.23.0/docs/consumers/pkg-promptkit.md)
|
||||||
for the pinned protocol and retry details.
|
for protocol and retry details.
|
||||||
|
|
||||||
A model operation that exhausts structural repair, a live-parent local operation
|
A model operation that exhausts structural repair, a live-parent local operation
|
||||||
timeout, capacity rejection, transport failure, malformed provider response,
|
timeout, capacity rejection, transport failure, malformed provider response,
|
||||||
@@ -31,4 +31,4 @@ continue. Caller cancellation, broken assets or configuration, and unknown
|
|||||||
internal failures remain fatal and cancel peers. Audita decodes into domain
|
internal failures remain fatal and cancel peers. Audita decodes into domain
|
||||||
types only after PromptKit validates the JSON. Ordinary reports use safe reason
|
types only after PromptKit validates the JSON. Ordinary reports use safe reason
|
||||||
codes and counts rather than provider messages. Full request/response content
|
codes and counts rather than provider messages. Full request/response content
|
||||||
is written only with `--debug`; see [operations](../operations.md#diagnostics-directory).
|
is written only with `--debug`; see [operations](../operations.md#debug-evidence).
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
The corrected transcript uses the selected `output.schema`: `bare-segments`
|
The corrected transcript uses the selected `output.schema`: `bare-segments`
|
||||||
or `audita-v1`. Its schema and text rules are unchanged by resilience
|
or `audita-v1`. Its schema and text rules are unchanged by resilience
|
||||||
reporting. See [configuration](../config.md#settings) for selection and
|
reporting. See [configuration](../config.md#settings) for selection and
|
||||||
[operations](../operations.md#required-output-files) for placement and writes.
|
[operations](../operations.md#outputs-and-lifecycle) for placement and writes.
|
||||||
|
|
||||||
With reporting enabled, Audita produces three ordinary files:
|
With reporting enabled, Audita produces three ordinary files:
|
||||||
|
|
||||||
@@ -46,24 +46,34 @@ remain in opt-in debug files.
|
|||||||
## Error and Recovery Records
|
## Error and Recovery Records
|
||||||
|
|
||||||
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
|
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
|
||||||
and `chunk_index`, module key/instance, initial segment range, applied-correction
|
and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction
|
||||||
count, and a `failures` sequence. Each failure identifies the operation
|
count, and a `failures` sequence. Each failure identifies the operation
|
||||||
(`proposal`, `validation`, or `validation_input`), optional validator and
|
(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
|
||||||
zero-based batch index, safe reason/message, sorted proposal indices when
|
zero-based batch index, safe reason/message, sorted proposal indices when
|
||||||
known, and execution counts when available. Multiple failed batches in one
|
known, and execution counts when available. Multiple failed batches in one
|
||||||
work unit form one error record. `scope: run` instead records phase, safe reason
|
work unit form one error record. A record with scope `run` records a fatal
|
||||||
and message, optional origin, and output path where relevant. It can describe
|
processing failure or a separate output delivery failure. It identifies the
|
||||||
a fatal error or secondary output-write failure without inventing a failed
|
phase, safe reason and message, and available origin. A processing origin can
|
||||||
model work unit. Raw provider errors and transcript text are absent.
|
include the operation, validator, batch, proposal index, and repair attempt.
|
||||||
|
A delivery failure includes the output path when known. The initiating
|
||||||
|
processing failure can include an `interruptions` sequence with each
|
||||||
|
available peer's phase, safe reason and message, and available origin.
|
||||||
|
Peer details do not add to the recoverable work-unit or run-error counts.
|
||||||
|
The work-unit matrix marks interrupted work. Raw provider errors and
|
||||||
|
transcript text are absent.
|
||||||
For `validation_decisions_invalid`, proposal indices identify only candidates
|
For `validation_decisions_invalid`, proposal indices identify only candidates
|
||||||
whose required decisions were unavailable. Other candidates in the same batch
|
whose required decisions were unavailable. Other candidates in the same batch
|
||||||
may still be approved or rejected normally.
|
may still be approved or rejected normally.
|
||||||
|
|
||||||
Each recovery identifies its work unit and successful operation, with sorted
|
Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted
|
||||||
`reasons`, a safe message, available execution counts, and sorted
|
`reasons`, a safe message, available execution counts, and sorted
|
||||||
`filtered_proposals` details. One recovery can contain `output_repaired`,
|
`filtered_proposals` details. One recovery can contain `output_repaired`,
|
||||||
`service_recovered`, and `proposals_filtered` together. Filtering covers
|
`service_recovered`, `correction_revised`, and `proposals_filtered` together.
|
||||||
defective response proposals; low confidence, protected-term rejection,
|
`correction_revised` means a repair yielded a fully validated revision; its
|
||||||
|
application outcome is recorded in the Markdown report. Filtering covers
|
||||||
|
defective proposals and revisions under their originating operation. Only
|
||||||
|
original proposal operations count toward `proposal_responses_with_filtering`.
|
||||||
|
Protected-term rejection,
|
||||||
ordinary editorial rejection, and an apply-time conflict caused by an earlier
|
ordinary editorial rejection, and an apply-time conflict caused by an earlier
|
||||||
accepted correction are not filtering recoveries. Failed operations are errors,
|
accepted correction are not filtering recoveries. Failed operations are errors,
|
||||||
not recoveries. The initial chunk identity remains stable even if earlier
|
not recoveries. The initial chunk identity remains stable even if earlier
|
||||||
@@ -79,23 +89,29 @@ is an error even when unexpected indices were also ignored.
|
|||||||
`report.md` includes run ID, status, corrections, work coverage, error and
|
`report.md` includes run ID, status, corrections, work coverage, error and
|
||||||
recovery totals, input/output paths, and full work-unit error and recovery
|
recovery totals, input/output paths, and full work-unit error and recovery
|
||||||
matrices. Rows show every initial chunk with 1-based chunk number and initial
|
matrices. Rows show every initial chunk with 1-based chunk number and initial
|
||||||
segment range; columns follow configured module-instance order, including
|
segment range; columns follow configured pipeline order, including repeated modules. Labels show
|
||||||
repeats. Completed error cells are `0` or `1`; recovery cells are operation
|
the one-based position and module, plus the operator ID when supplied. Completed error cells are `0` or `1`; recovery cells are operation
|
||||||
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
counts. `—` means not started and `n*` means interrupted with n facts recorded.
|
||||||
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
|
||||||
records containing each reason; a recovery can have several reasons, so those
|
records containing each reason; a recovery can have several reasons, so those
|
||||||
counts need not sum to the recovery total. The report refers to both YAML
|
counts need not sum to the recovery total. When a repair was attempted, a
|
||||||
files. It does not include transcript excerpts or correction lists.
|
Correction repair table lists its stage, chunk,
|
||||||
|
original proposal index, semantic attempt count, and final outcome (`applied`,
|
||||||
|
`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or
|
||||||
|
`interrupted`). The report refers to both YAML files. It does not include
|
||||||
|
transcript excerpts or correction lists.
|
||||||
|
|
||||||
## Debug-Only Files
|
## Debug-Only Files
|
||||||
|
|
||||||
With `--debug`, `correction-ledger.json` contains proposed candidates and
|
With `--debug`, `correction-ledger.json` contains proposed candidates and
|
||||||
their application, rejection, or interruption details. `llm-execution.json`
|
their application, rejection, or interruption details. `llm-execution.json`
|
||||||
holds logical calls and PromptKit execution events; `utilization-diagnostics.json`
|
holds logical calls and PromptKit execution events. Interaction request/response
|
||||||
groups backend capacity and timing observations while preserving known and
|
artifacts and input snapshots are also debug-only. Records carry zero-based
|
||||||
complete usage semantics. Interaction request/response artifacts, selected
|
`stage_index` and optional `stage_id` where an entry is identified.
|
||||||
profiles, configuration provenance, and input snapshots are also debug-only.
|
|
||||||
These files are diagnostic evidence, not an alternate ordinary report schema.
|
These files are diagnostic evidence, not an alternate ordinary report schema.
|
||||||
|
For a repaired proposal, one ledger row preserves the original proposed edit,
|
||||||
|
the final applied segment and span when applicable, and repair attempts with
|
||||||
|
revision decisions and interaction artifact paths.
|
||||||
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
|
||||||
literal replacement span, even when it is `""` for a deletion. Applied entries
|
literal replacement span, even when it is `""` for a deletion. Applied entries
|
||||||
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
also include `applied_corrected_text`, likewise `""` for an applied deletion.
|
||||||
|
|||||||
@@ -6,35 +6,51 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
|
|||||||
|
|
||||||
## Invocation
|
## Invocation
|
||||||
|
|
||||||
Use an argument array with the transcript path and required glossary argument,
|
Use an argument array with the transcript path and configured glossary selections,
|
||||||
following the [process examples](../cli.md#commands-and-help). For machine workflows,
|
following the [process examples](../cli.md#processing). For machine workflows,
|
||||||
select an explicit transcript file destination and a configuration path. Reports
|
select an explicit transcript file destination and a configuration path. Reports
|
||||||
then go beside the transcript. For stdout output, select `--report-dir` or use
|
go beside the transcript unless an explicit `--report-dir` selects another
|
||||||
the invocation working directory. Debug is independent and off by default.
|
directory. For stdout output, reports use the effective `reporting.stdout_dir`
|
||||||
Supplying a configuration path also
|
(defaulting to the invocation working directory), including a `--report-dir`
|
||||||
makes file selection predictable; only explicitly referenced credential variables
|
override. A config-file or overlay value for `reporting.stdout_dir` alone does
|
||||||
are read from the environment.
|
not redirect file-output reports. `--work-dir` selects a debug root, while
|
||||||
|
`--debug` enables capture; the path flag alone does not.
|
||||||
|
Supplying a configuration path also makes file selection predictable; only
|
||||||
|
explicitly referenced credential variables are read from the environment.
|
||||||
|
|
||||||
|
Choose a trusted configuration bundle. An explicit main path still allows
|
||||||
|
its configured imports and automatic overlay, glossary, and prompt companions.
|
||||||
|
Without a main path, Audita uses the configured discovery locations. The
|
||||||
|
parent controls who can write those sources and which report destinations and
|
||||||
|
retained debug directories the invocation will use. Native inspection is
|
||||||
|
separate from processing preflight.
|
||||||
|
|
||||||
Resolve the executable deliberately and set the child's working directory so
|
Resolve the executable deliberately and set the child's working directory so
|
||||||
relative input/output paths and reported artifact paths have a known base.
|
relative input/output paths and reported artifact paths have a known base.
|
||||||
Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout
|
Read stdout and stderr concurrently to avoid pipe blocking, and impose timeout
|
||||||
or cancellation in the parent process when needed.
|
or cancellation in the parent process when needed. Optional `--progress`
|
||||||
|
messages arrive on stderr during stage and chunk execution, including model
|
||||||
|
operations and completions. Debug enables progress by default, and
|
||||||
|
`--progress=false` or quiet mode suppresses it. Consume progress as
|
||||||
|
human-readable telemetry. Stdout remains the transcript stream.
|
||||||
|
|
||||||
## Result Handling
|
## Result Handling
|
||||||
|
|
||||||
Interpret the exit result according to the
|
Interpret the exit result according to the
|
||||||
[CLI output and exit contract](../cli.md#streams-and-exit-status).
|
[CLI output and exit contract](../cli.md#inspection-and-outcomes).
|
||||||
Consume transcript/report data through the
|
Consume transcript/report data through the
|
||||||
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
[output-file contract](output-files.md), and treat stderr as human-readable detail.
|
||||||
The three reports may be absent after early configuration or destination
|
Audita loads configuration and parses the input transcript before report
|
||||||
failures. After initialization, failed reports can describe invalid input or
|
initialization. Early destination failures can also leave no reports. After
|
||||||
partial processing. Exit `0` can include recoverable model errors and applied
|
initialization, failed reports can describe partial processing or output
|
||||||
|
failures and use the same report directory as successful runs.
|
||||||
|
Exit `0` can include recoverable model errors and applied
|
||||||
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
|
||||||
transcript may have been written before a later required report failure, so
|
transcript may have been written before a later required report failure, so
|
||||||
the file alone does not prove completion.
|
the file alone does not prove completion.
|
||||||
|
|
||||||
If processing fails, preserve available evidence and follow
|
If processing fails, preserve available evidence and follow
|
||||||
[operations recovery](../operations.md#failure-inspection-and-recovery).
|
[operations recovery](../operations.md#recovery).
|
||||||
Ordinary reports are independent of optional retained debug evidence. Parent
|
Ordinary reports are independent of optional retained debug evidence. Parent
|
||||||
cancellation reaches the child through its context; Audita joins workers and
|
cancellation reaches the child through its context; Audita joins workers and
|
||||||
attempts failed-status report finalization before exiting. Uncatchable
|
attempts failed-status report finalization before exiting. Uncatchable
|
||||||
@@ -45,4 +61,4 @@ file nor a debug directory alone establishes successful completion.
|
|||||||
|
|
||||||
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
- [Transcript and glossary inputs](transcript-glossary-files.md)
|
||||||
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
- [LLM endpoint compatibility](openai-compatible-llm.md)
|
||||||
- [Configuration inspection commands](../cli.md#commands-and-help)
|
- [Configuration inspection commands](../cli.md#inspection-and-outcomes)
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ schema/version marker.
|
|||||||
|
|
||||||
| Segment field | Type and validation |
|
| Segment field | Type and validation |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `id` | Optional integer (or null). Supplied IDs must be unique; missing IDs are accepted. |
|
| `id` | Optional integer (or null) in the Go `int` range for the executable's architecture. Supplied IDs are compared exactly and must be unique. |
|
||||||
| `speaker` | Non-empty string |
|
| `speaker` | Non-empty string |
|
||||||
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
|
| `start`, `end` | Numeric seconds, finite and non-negative; end must be at least start |
|
||||||
| `text` | Non-empty string |
|
| `text` | Non-empty string |
|
||||||
@@ -21,30 +21,18 @@ schema/version marker.
|
|||||||
|
|
||||||
Supply explicit start and end times for meaningful timing. The current decoder
|
Supply explicit start and end times for meaningful timing. The current decoder
|
||||||
uses zero for missing or null timestamps rather than enforcing their presence.
|
uses zero for missing or null timestamps rather than enforcing their presence.
|
||||||
String validation checks empty values, not whitespace trimming. Normalization can
|
String validation checks for empty values. Normalization can merge segments and
|
||||||
merge segments and reassign IDs, so source IDs need not match output IDs.
|
reassign IDs, so source IDs need not match output IDs.
|
||||||
|
|
||||||
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
|
The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyable
|
||||||
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
|
||||||
types, duplicate supplied IDs, and the validation failures above are rejected.
|
types, duplicate supplied IDs, and the validation failures above are rejected.
|
||||||
|
|
||||||
## Glossary YAML Contract
|
## Glossary collections
|
||||||
|
|
||||||
The top-level object contains a `glossary` list with at least one entry.
|
Glossary definitions are composed as application configuration, including the
|
||||||
|
optional companion inputs described by the [configuration reference](../config.md).
|
||||||
| Entry field | Type and validation |
|
The [local example](../../examples/local/glossary.yml) provides a complete
|
||||||
| --- | --- |
|
collection. Configuration may contain empty collections; processing preflight
|
||||||
| `name` | Non-empty string |
|
requires every glossary-correction entry to select at least one glossary entry.
|
||||||
| `category` | Non-empty string |
|
The previous standalone `glossary` file format is not a processing input.
|
||||||
| `summary` | Non-empty string |
|
|
||||||
| `aliases` | Optional list of non-empty strings |
|
|
||||||
| `plural` | Optional string |
|
|
||||||
|
|
||||||
The parser checks empty strings without trimming them. Unknown YAML fields are
|
|
||||||
ignored by the glossary parser, unlike the stricter
|
|
||||||
[configuration-file schema](../config.md#loading-and-precedence).
|
|
||||||
|
|
||||||
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
|
|
||||||
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
|
|
||||||
and empty aliases are rejected. These formats describe input acceptance, not
|
|
||||||
additional vocabulary restrictions.
|
|
||||||
|
|||||||
@@ -9,13 +9,12 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
|
|||||||
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
1. Update fields, defaults, loading, and validation in `internal/core/config`.
|
||||||
2. Update file, selected overlay, credential reference, and CLI mapping as
|
2. Update file, selected overlay, credential reference, and CLI mapping as
|
||||||
applicable. Preserve the precedence defined in the
|
applicable. Preserve the precedence defined in the
|
||||||
[configuration contract](../config.md#loading-and-precedence), including the
|
[configuration contract](../config.md#loading-and-composition), including the
|
||||||
distinct validation and effective-config commands.
|
distinct validation and effective-config commands.
|
||||||
3. Update redaction when secrets are involved.
|
3. Review configuration tests and related CLI tests for precedence, validation,
|
||||||
4. Review configuration tests and related CLI tests for precedence, validation,
|
|
||||||
and source-specific behavior. Add or adjust tests for the changed contract
|
and source-specific behavior. Add or adjust tests for the changed contract
|
||||||
according to the [testing policy](../policy/testing.md).
|
according to the [testing policy](../policy/testing.md).
|
||||||
5. Update the configuration reference and affected [examples](../../examples/).
|
4. Update the configuration reference and affected [examples](../../examples/).
|
||||||
|
|
||||||
## Changing CLI Behavior
|
## Changing CLI Behavior
|
||||||
|
|
||||||
@@ -27,23 +26,27 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
|
|||||||
for the changed behavior.
|
for the changed behavior.
|
||||||
4. Update the CLI reference and affected integration guidance.
|
4. Update the CLI reference and affected integration guidance.
|
||||||
|
|
||||||
|
The `version` route uses ReleaseKit's public `buildinfo.Version()` accessor and
|
||||||
|
stays independent of application runtime assembly. Preserve its behavior from
|
||||||
|
the [CLI contract](../cli.md) when changing the command; the
|
||||||
|
adoption decision is recorded in [ADR 0001](../adr/0001-releasekit.md).
|
||||||
|
|
||||||
|
Processing preflight checks glossary selections before inspecting proposal,
|
||||||
|
validation, and enabled repair profiles in pipeline order. The checked plan
|
||||||
|
carries the resolved module specs and glossary views into execution. It reuses
|
||||||
|
profile and backend inspections and equivalent glossary views within the
|
||||||
|
invocation. Native inspection commands continue to inspect configuration
|
||||||
|
without running processing readiness checks.
|
||||||
|
|
||||||
## Validating Examples
|
## Validating Examples
|
||||||
|
|
||||||
Keep complete copyable files in [examples](../../examples/) and link to them from
|
Keep complete copyable files in [examples](../../examples/) and link to them from
|
||||||
their canonical contract documentation. Keep them valid and secret-free.
|
their canonical contract documentation. Keep them valid and secret-free.
|
||||||
From the repository root, validate the maintained configuration examples:
|
Run the commands in the [example guide](../../examples/README.md) from each
|
||||||
|
project's directory. `internal/cli/examples_integration_test.go` exercises the
|
||||||
```sh
|
maintained projects with a controlled provider, and the preview fixture can be
|
||||||
go run ./cmd/audita config validate --config examples/minimal-config.yml
|
prepared without inference. Review automatic companion sources and overlays in
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
|
their actual project directories. Keep service credentials synthetic in tests.
|
||||||
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
|
|
||||||
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
|
|
||||||
```
|
|
||||||
|
|
||||||
Only referenced credentials read environment values; use a controlled synthetic
|
|
||||||
value when validating the service example. Check affected CLI examples against the
|
|
||||||
parser and use schema tests for maintained transcript/glossary inputs. Example
|
|
||||||
validation should not invoke a live LLM merely to check documentation.
|
|
||||||
|
|
||||||
## Runtime Change Checks
|
## Runtime Change Checks
|
||||||
|
|
||||||
@@ -54,8 +57,7 @@ Select checks according to the [testing policy](../policy/testing.md):
|
|||||||
- Process/output changes: verify stdout, stderr, exit, output-file, and report
|
- Process/output changes: verify stdout, stderr, exit, output-file, and report
|
||||||
behavior against the [CLI contract](../cli.md) and
|
behavior against the [CLI contract](../cli.md) and
|
||||||
[subprocess guidance](../integrations/subprocess.md), including failures.
|
[subprocess guidance](../integrations/subprocess.md), including failures.
|
||||||
- Report or diagnostics changes: review [report assembly tests and redaction
|
- Report or diagnostics changes: review [report assembly tests](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
|
||||||
checks](diagnostics-reporting.md#key-tests), metadata fields, and artifact paths.
|
|
||||||
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
|
- Schema or serialization changes: review [output-schema tests](output-schemas.md#key-tests)
|
||||||
and `internal/core/schema` tests, including supported formats and rejection of
|
and `internal/core/schema` tests, including supported formats and rejection of
|
||||||
unsupported schemas.
|
unsupported schemas.
|
||||||
|
|||||||
@@ -12,8 +12,7 @@ work-unit errors, separate run errors, and one recovery per successful logical
|
|||||||
operation. Reports order stages, sections, validators, and batches by semantic
|
operation. Reports order stages, sections, validators, and batches by semantic
|
||||||
identity, never concurrent completion time. The core reporting package owns
|
identity, never concurrent completion time. The core reporting package owns
|
||||||
Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI
|
Audita's versioned YAML DTOs, fixed Markdown rendering, and brief CLI
|
||||||
completion formatter. YAML redaction walks serialized string nodes so numeric
|
completion formatter. Reports encode the typed documents directly to YAML. Ordinary reports omit raw causes,
|
||||||
and boolean values retain their types. Ordinary reports omit raw causes,
|
|
||||||
transcript text, complete decisions, and token/timing payloads.
|
transcript text, complete decisions, and token/timing payloads.
|
||||||
Validation operations with unavailable decisions report only affected requested
|
Validation operations with unavailable decisions report only affected requested
|
||||||
indices, even when other candidates in the batch continue. Ignored extra
|
indices, even when other candidates in the batch continue. Ignored extra
|
||||||
@@ -21,39 +20,53 @@ validator entries yield `validation_decisions_filtered` only for a successful
|
|||||||
operation; they do not count as filtered correction proposals. Failed operations
|
operation; they do not count as filtered correction proposals. Failed operations
|
||||||
retain their execution counts and never produce a recovery record.
|
retain their execution counts and never produce a recovery record.
|
||||||
|
|
||||||
`internal/core/diagnostics.OutputSession` owns active destination checks,
|
`internal/core/diagnostics.OutputSession` owns destination selection, report
|
||||||
transcript preflight, report initialization, regular-file replacement, and direct
|
initialization, and publication. Transcript and report files share a staging
|
||||||
stream publication. Transcript input is protected from early report writes but
|
primitive that writes beside the resolved destination. Streams are opened for
|
||||||
may be the final transcript destination. The LLM runtime records successful reads
|
delivery. Transcript input is protected from early report writes but
|
||||||
through PromptKit's profile filesystem and supplies those source paths to CLI;
|
may be the final transcript destination. The LLM runtime records consumed PromptKit sources and effective host message
|
||||||
|
files and supplies those source paths to CLI;
|
||||||
no second filename-based discovery rule is used.
|
no second filename-based discovery rule is used.
|
||||||
CLI composes it after configuration/profile inspection and before reading
|
CLI composes it after configuration, transcript acquisition, and preflight
|
||||||
inputs or dispatching model work. It does not create ordinary report parents.
|
readiness and before dispatching model work. It does not create ordinary report parents.
|
||||||
On a final write error it attempts other enabled reports and one failed-status
|
Final report rendering includes delivery and cleanup errors observed so far.
|
||||||
pass; a published transcript is never erased by a later report failure.
|
When another failure becomes known, later writes use updated failed reports.
|
||||||
|
One best-effort pass refreshes earlier reports with stale evidence. A published
|
||||||
|
transcript remains available after a later report failure.
|
||||||
|
|
||||||
Debug is independent. Only `diagnostics.debug` creates a unique retained child
|
Debug is independent. The shared `--debug` flag enables a unique retained child
|
||||||
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
|
under `diagnostics.work_dir`. The CLI creates one interaction sink rooted in
|
||||||
when debug is off. With debug on, source/normalized snapshots, invocation,
|
that directory and passes it to proposal generation, validation, and repair.
|
||||||
effective settings, origins, profiles, initial chunk summary, correction ledger,
|
With debug enabled, it saves the source bytes as acquired, separate from the parsed and normalized
|
||||||
indexed execution/utilization, and model interactions are written. Runner
|
transcript snapshots. It also captures invocation metadata, effective settings
|
||||||
candidate capture includes proposals later rejected or interrupted. Ledger
|
rendered by ConfigKit, the correction ledger, LLM execution observations, and
|
||||||
mapping uses optional string pointers so an empty proposed replacement is
|
model interactions. Candidate capture includes proposals later
|
||||||
serialized for every candidate and an empty applied replacement appears only
|
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
|
||||||
when that candidate applied; see the [public artifact contract](../integrations/output-files.md#debug-only-files).
|
attempt, validator, and batch coordinates as applicable. Request metadata records
|
||||||
Debug write failures reach the CLI through an explicit concurrency-safe diagnostic
|
these coordinates with the interaction role. Proposal request metadata carries
|
||||||
callback and produce redacted stderr messages without aborting processing.
|
the zero-based response origin (`start_index`) and the ordered `segment_ids`
|
||||||
Completed validator decisions returned with a fatal batch error remain available
|
used to interpret response references. Each interaction retains its available
|
||||||
to the debug ledger; they never authorize applying an incompletely validated edit.
|
prepared request, response, and failure evidence. Ledger mapping preserves empty
|
||||||
Runner decision records carry the validator's explicit approved, rejected, or
|
proposed and applied replacements and distinguishes unavailable candidates from
|
||||||
unavailable disposition into the ledger. Unavailable candidates have a distinct
|
interrupted work.
|
||||||
top-level ledger disposition, while fatal interruptions remain interrupted when
|
While final proposal indices are pending, repair request snapshots share their
|
||||||
no rejection or unavailable decision was returned. Successful validation
|
unchanged message prefix. Each written request contains its full history.
|
||||||
interaction payloads contain the full decoded response, final content, and
|
|
||||||
reconciliation outcomes; ignored indices do not become ledger candidates.
|
The sink groups concurrent write failures by role, artifact operation, and cause.
|
||||||
Every debug writer uses all configured backend secrets for redaction, including
|
Warnings report the count and an affected path. The writer still attempts other
|
||||||
unselected backend credentials and source snapshots. Debug can still contain
|
artifacts and returns the paths it wrote, and the CLI presents warnings separately
|
||||||
sensitive transcript and prompt content.
|
from processing outcomes and required delivery. Debug artifacts can contain sensitive transcript and
|
||||||
|
prompt content. Execution
|
||||||
|
position and optional operator ID are carried separately from generated safe
|
||||||
|
artifact path coordinates.
|
||||||
|
|
||||||
|
Execution diagnostics include a native-group invocation summary and per-call
|
||||||
|
summaries. Terminal coverage is separate from usage completeness; unknown usage
|
||||||
|
is distinct from known zero. Overlapping summaries must not be added together.
|
||||||
|
The collector snapshots after work joins and retains domain labels and stable
|
||||||
|
presentation order. Prepared evidence comes from `OnPrepared`, so preparation or
|
||||||
|
admission failures may have no request snapshot. Coordinator execution failures
|
||||||
|
still finalize the attempted logical call and its available evidence.
|
||||||
|
|
||||||
## Key Tests
|
## Key Tests
|
||||||
|
|
||||||
@@ -61,3 +74,17 @@ Useful tests: `internal/core/reporting/*_test.go`,
|
|||||||
`internal/core/diagnostics/*_test.go`,
|
`internal/core/diagnostics/*_test.go`,
|
||||||
`internal/framework/processreport/*_test.go`, and
|
`internal/framework/processreport/*_test.go`, and
|
||||||
`cmd/audita/main_integration_test.go`.
|
`cmd/audita/main_integration_test.go`.
|
||||||
|
|
||||||
|
Report projection uses typed operation and delivery results. Recoverable
|
||||||
|
operations count toward work-unit errors. A fatal processing cause retains its
|
||||||
|
own phase and available operation coordinates, and peer interruptions and later
|
||||||
|
file failures preserve their own attribution. The correction ledger joins one
|
||||||
|
candidate row by module instance and proposal index, preserving repair lineage
|
||||||
|
and the recorded validator execution class.
|
||||||
|
|
||||||
|
The invocation selects retained evidence once. Summary-only processing
|
||||||
|
accumulates counters as completed sections leave the work window. Ordinary
|
||||||
|
reports retain the full work matrix and the operation facts that contribute to
|
||||||
|
errors and recoveries. Debug retains
|
||||||
|
candidate and decision history, repair lineage, and native observations.
|
||||||
|
Without debug, the runtime skips prepared message snapshots.
|
||||||
|
|||||||
@@ -1,39 +1,66 @@
|
|||||||
# Audita LLM Runtime
|
# Audita LLM Runtime
|
||||||
|
|
||||||
## Client Boundary
|
`llm.NewRuntime` assembles one captured PromptKit engine using its default
|
||||||
|
catalog, Audita's embedded prompts and schemas, and optional operator sources
|
||||||
|
anchored to the recorded application main context. Captured host sources and
|
||||||
|
message files are passed to output collision checks. An embedded-only diagnostic
|
||||||
|
construction distinguishes shipped asset failures from operator-source failures;
|
||||||
|
it never replaces the requested engine.
|
||||||
|
|
||||||
`llm.NewRuntime` assembles one PromptKit engine for both roles. It loads embedded
|
Each resolved pipeline entry receives role-bound clients with its selected
|
||||||
prompt and schema assets, registers application backends, and inspects the
|
profiles and effective budgets. Clients share engine-local backend admission.
|
||||||
selected proposal and validation profiles offline. A role-bound `llm.RoleClient`
|
The adapter uses ordinary `Run` with `OperationTimeout` and `OnPrepared`, passes
|
||||||
implements `structuredoutput.Client`; proposal and validator contracts alias that
|
Audita's per-call schema and repair budget, and decodes only validated results.
|
||||||
interface. Both roles share engine-local backend capacity.
|
The operation budget includes preparation, admission, retries, and structural
|
||||||
|
repair. An expired parent context stops processing and retains any native cause.
|
||||||
|
With an active parent, PromptKit's operation deadline remains recoverable even
|
||||||
|
when a preparation error wraps it. A native transport timeout affects the HTTP
|
||||||
|
attempt and is recorded as a transport failure. The prepared notification
|
||||||
|
supplies prompt metadata to observers. With debug capture enabled, it also
|
||||||
|
supplies the actual request messages when preparation reaches that point.
|
||||||
|
PromptKit owns backend and credential selection. When a required credential is absent,
|
||||||
|
the CLI identifies the role, profile, and environment variable using the native
|
||||||
|
diagnostic. See the [integration contract](../integrations/openai-compatible-llm.md)
|
||||||
|
for failure behavior and the [configuration reference](../config.md) for sources.
|
||||||
|
|
||||||
For each logical call, the adapter starts an operation context and a diagnostic
|
When correction repair is enabled, the runner checks the module's repair prompt
|
||||||
record before preparation. It supplies the reserved hardening text, the selected
|
and coordinator before submitting proposal work. The proposal call captures a
|
||||||
profile and resolved backend credential, and the complete embedded schema/repair
|
native immutable PromptKit exchange. Each actionable correction starts an independent
|
||||||
contract. `PrepareExecution` produces the exact messages and hashes retained in
|
`Engine.Coordinate` branch from that exchange. PromptKit assembles continuation
|
||||||
an Audita snapshot. The adapter checks the prepared backend and endpoint before
|
history, owns the additional-turn budget, and delivers the last permitted response
|
||||||
`RunPrepared`. It records available final content, usage, model, and profile
|
for assessment. Follow-ups select the repair prompt, profile, and output schema
|
||||||
metadata and decodes only a validated result. Call hooks expose logical
|
directly; Audita does not rerender the original proposal or assemble replay messages.
|
||||||
start/finish records and PromptKit execution events for debug utilization.
|
An opaque exchange in the structured-output contract keeps provider state outside
|
||||||
|
modules and validators. The adapter shares request construction, notifications,
|
||||||
|
observers, and once-only completion between ordinary and coordinated calls.
|
||||||
|
Completion attaches terminal counts when available and records one logical finish.
|
||||||
|
Progress uses the adapter's logical call boundaries. Writes to the diagnostic
|
||||||
|
stream occur outside PromptKit's operation deadline.
|
||||||
|
Prepared requests and terminal output travel in the returned response for an
|
||||||
|
optional interaction artifact. Logical call notifications carry metadata.
|
||||||
|
|
||||||
The [integration contract](../integrations/openai-compatible-llm.md) describes
|
Audita owns actionable rejection, revision decoding, withdrawal, current previews,
|
||||||
wire ownership and failure behavior. `structuredoutput.Failure` has a safe
|
and full revalidation. Each branch returns its attempt facts and final approval
|
||||||
message, fixed reason code, typed cause, and recoverable/fatal/canceled class.
|
to the section runner. Accepted siblings remain independent. Requesting more work
|
||||||
Counts distinguish provider attempts, transient retries, and structural
|
at the budget boundary produces exhaustion without allocating another logical call.
|
||||||
repairs, including on errors. Secrets from all configured backends, including unselected
|
A canceled, undelivered response supplies diagnostic evidence only; it cannot enter
|
||||||
ones, are available to the diagnostics redaction boundary.
|
the approved set. Repair disabled in configuration bypasses coordination entirely.
|
||||||
|
The runner supplements each repair instruction with its current proposal preview
|
||||||
|
as described in the [module guide](modules.md#failure-and-outcome-behavior).
|
||||||
|
The bundled prompts separate durable system principles from the initial user
|
||||||
|
assignment so later repair instructions can allow acceptable minor cleanup;
|
||||||
|
see [prompt instruction ownership](prompts.md).
|
||||||
|
|
||||||
## Orchestration
|
The observation collector tracks logical calls and execution events, and enrolls
|
||||||
|
selected runs in a native `ExecutionGroup`. Proposal, repair, and nested validator
|
||||||
|
clients explicitly share the invocation observer; history reuse adds no extra run.
|
||||||
|
Native summaries deduplicate terminal totals by run ID and retain terminal coverage
|
||||||
|
separately from usage presence and completeness. Logical calls and run membership
|
||||||
|
can differ, especially on failures before execution.
|
||||||
|
|
||||||
CLI processing supplies two role clients from the same engine to the runner.
|
Processing operations retain the failure class, reason, and underlying error.
|
||||||
PromptKit's per-backend admission applies to both roles and every generation,
|
When a client omits the reason, the operation uses `internal_failure` and keeps
|
||||||
retry, and repair. The runner still processes module instances in order,
|
the original class and error chain. Ordinary reports use compact outcomes.
|
||||||
collects concurrent section work deterministically, and cancels remaining
|
Request and response evidence is optional debug material. The runner preserves
|
||||||
section work only on fatal failures. `llm.DiagnosticsWriter` remains the protected
|
ordered pipeline traversal and deterministic section collection while canceling
|
||||||
request/response artifact writer. `llm.ObservationCollector` captures metadata
|
and joining peers on fatal errors.
|
||||||
events under a short mutex and freezes them after runner work joins. Its logical
|
|
||||||
records include preparation failures; summary usage comes from terminal events
|
|
||||||
once per call and retains knownness and completeness for each field. Runner wall
|
|
||||||
and validator timings remain separate from provider attempts. Ordinary reports
|
|
||||||
derive compact runner outcomes; full observations are debug-only.
|
|
||||||
|
|||||||
@@ -13,7 +13,13 @@ Modules implement `contracts.TranscriptModule`:
|
|||||||
- `Validators()`
|
- `Validators()`
|
||||||
- `Propose(ctx, req)`
|
- `Propose(ctx, req)`
|
||||||
|
|
||||||
Runner resolves configured module specs to module instances through `internal/framework/modules`.
|
Built-in modules also expose `RepairPromptID()` for the runner to select their
|
||||||
|
repair definitions; proposal-only callers retain the existing interface.
|
||||||
|
|
||||||
|
The CLI supplies the built-in factory from `internal/modules` to the runner.
|
||||||
|
The runner resolves checked module specs through that factory. Module instances
|
||||||
|
can receive concurrent proposal calls across sections. Request inputs are
|
||||||
|
borrowed for the call and must be copied before modification or retention.
|
||||||
|
|
||||||
## Built-In Modules
|
## Built-In Modules
|
||||||
|
|
||||||
@@ -37,13 +43,13 @@ Shared proposal-generation plumbing is centralized in:
|
|||||||
Module packages own:
|
Module packages own:
|
||||||
- prompt selection (`internal/prompts` prompt IDs).
|
- prompt selection (`internal/prompts` prompt IDs).
|
||||||
|
|
||||||
`proposal_generation.BuildProposalVariables` encodes the glossary, selected
|
Each stage prepares the encoded glossary context and protected spellings once.
|
||||||
transcript section, and trimmed description once for all four modules. PromptKit
|
Proposal generation adds the selected transcript section and trimmed description
|
||||||
renders their embedded definitions from those variables and validates the
|
for each request. PromptKit renders the selected definition and validates the
|
||||||
correction-set schema before Audita maps proposals. Domain validators still
|
correction-set schema before Audita maps proposals. Domain validators still
|
||||||
check confidence, applicability, protected glossary terms, and replacement
|
check applicability, protected glossary terms, and replacement
|
||||||
policy. Model-facing segment IDs are one-based positions within the proposal
|
policy. Model-facing segment IDs are one-based positions within the proposal
|
||||||
request. Generation maps returned references back to the current chunk's actual
|
request. Generation and repair map those references back to the current chunk's actual
|
||||||
segment IDs, preserving response order and multiple proposals per segment.
|
segment IDs, preserving response order and multiple proposals per segment.
|
||||||
Unknown references become invalid-target candidates for the existing domain
|
Unknown references become invalid-target candidates for the existing domain
|
||||||
checks; they are never interpreted as actual transcript IDs. Debug request
|
checks; they are never interpreted as actual transcript IDs. Debug request
|
||||||
@@ -61,6 +67,17 @@ Module packages call the built-in chain resolver at construction.
|
|||||||
|
|
||||||
## Failure and Outcome Behavior
|
## Failure and Outcome Behavior
|
||||||
|
|
||||||
|
Each actionable rejection can be revised or withdrawn within its original
|
||||||
|
section. A revision retains the original proposal identity and must pass the
|
||||||
|
full validator chain before application.
|
||||||
|
|
||||||
|
Before each repair attempt, the runner derives the current proposal's full
|
||||||
|
before/after text with the same domain preview used by validation. It refreshes
|
||||||
|
the view after revisions, including changes of target segment. A failed preview
|
||||||
|
retains available original text and findings without supplying a resulting
|
||||||
|
segment. This context supplements the original exchange and does not mutate the
|
||||||
|
transcript. The model-facing fields are described in the [prompt guide](prompts.md).
|
||||||
|
|
||||||
Module construction failures are fatal. Recoverable proposal model failures
|
Module construction failures are fatal. Recoverable proposal model failures
|
||||||
record an unsuccessful operation and yield no candidates for that section;
|
record an unsuccessful operation and yield no candidates for that section;
|
||||||
other sections and module instances continue. Fatal adapter or application
|
other sections and module instances continue. Fatal adapter or application
|
||||||
@@ -69,6 +86,11 @@ outcomes are defined by the [report contract](../integrations/output-files.md).
|
|||||||
|
|
||||||
## Changing Modules
|
## Changing Modules
|
||||||
|
|
||||||
|
Frequently reevaluate the default module combination and sequence against what
|
||||||
|
has been observed to work best for most workloads. Prior empirical results inform
|
||||||
|
the current choice without fixing a permanent number or order of passes. Maintain
|
||||||
|
the selected default in the [configuration contract](../config.md#pipeline-and-glossary-selections).
|
||||||
|
|
||||||
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
||||||
2. Keep module prompt selection in the module and its definition in
|
2. Keep module prompt selection in the module and its definition in
|
||||||
`internal/prompts`; see [prompt assets](prompts.md).
|
`internal/prompts`; see [prompt assets](prompts.md).
|
||||||
@@ -86,6 +108,6 @@ outcomes are defined by the [report contract](../integrations/output-files.md).
|
|||||||
## Key Tests
|
## Key Tests
|
||||||
|
|
||||||
- `internal/modules/*/module_test.go`
|
- `internal/modules/*/module_test.go`
|
||||||
- `internal/framework/modules/registry_test.go`
|
- `internal/modules/registry_test.go`
|
||||||
- `internal/framework/proposal_generation/*_test.go`
|
- `internal/framework/proposal_generation/*_test.go`
|
||||||
- `internal/cli/run_test.go` (pipeline/report integration)
|
- `internal/cli/process_ordered_integration_test.go` (pipeline/report integration)
|
||||||
|
|||||||
@@ -12,12 +12,14 @@ selectable values and defaults belong to
|
|||||||
|
|
||||||
Configuration validation checks registry support. CLI orchestration resolves the
|
Configuration validation checks registry support. CLI orchestration resolves the
|
||||||
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
selected encoder, passes the final transcript to it, and routes the encoded bytes
|
||||||
to the destination defined by the [CLI contract](../cli.md#streams-and-exit-status).
|
to the destination defined by the [CLI contract](../cli.md#inspection-and-outcomes).
|
||||||
An unsupported key fails validation or resolution rather than selecting an
|
An unsupported key fails validation or resolution rather than selecting an
|
||||||
implicit fallback.
|
implicit fallback.
|
||||||
|
|
||||||
The registry delegates bare segment serialization to the transcript schema
|
The registry delegates bare segment serialization to the transcript schema
|
||||||
package and uses an envelope encoder for the versioned form. Report construction
|
package and uses an envelope encoder for the versioned form. Both serialize
|
||||||
|
the model's typed segment array directly, including native integer IDs and
|
||||||
|
omitting empty categories. Report construction
|
||||||
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
|
receives the selected key separately; see [report assembly](diagnostics-reporting.md).
|
||||||
|
|
||||||
## Changing an Encoder
|
## Changing an Encoder
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ Executable and command orchestration:
|
|||||||
|
|
||||||
Core deterministic components:
|
Core deterministic components:
|
||||||
- `internal/core/config`
|
- `internal/core/config`
|
||||||
|
- `internal/core/glossary`
|
||||||
- `internal/core/schema`
|
- `internal/core/schema`
|
||||||
- `internal/core/normalization`
|
- `internal/core/normalization`
|
||||||
- `internal/core/chunking`
|
- `internal/core/chunking`
|
||||||
@@ -24,7 +25,7 @@ Core deterministic components:
|
|||||||
|
|
||||||
Framework orchestration and contracts:
|
Framework orchestration and contracts:
|
||||||
- `internal/framework/contracts`
|
- `internal/framework/contracts`
|
||||||
- `internal/framework/modules`
|
- `internal/framework/preflight`
|
||||||
- `internal/framework/proposal_generation`
|
- `internal/framework/proposal_generation`
|
||||||
- `internal/framework/proposals`
|
- `internal/framework/proposals`
|
||||||
- `internal/framework/runner`
|
- `internal/framework/runner`
|
||||||
@@ -37,6 +38,7 @@ Framework orchestration and contracts:
|
|||||||
- `internal/framework/stagename`
|
- `internal/framework/stagename`
|
||||||
|
|
||||||
Domain implementations:
|
Domain implementations:
|
||||||
|
- `internal/modules` (built-in module composition)
|
||||||
- `internal/modules/*`
|
- `internal/modules/*`
|
||||||
- `internal/validators/*`
|
- `internal/validators/*`
|
||||||
- `internal/prompts`
|
- `internal/prompts`
|
||||||
@@ -65,8 +67,8 @@ artifact lifecycle.
|
|||||||
|
|
||||||
## Component Responsibilities
|
## Component Responsibilities
|
||||||
|
|
||||||
Core components implement configuration defaults/loading/validation/redaction,
|
Core components implement configuration defaults, loading, and validation,
|
||||||
transcript and glossary schemas, normalization and chunking, output encoding,
|
transcript schemas and derived glossary views, normalization and chunking, output encoding,
|
||||||
diagnostics artifact helpers, and public report structures.
|
diagnostics artifact helpers, and public report structures.
|
||||||
|
|
||||||
Framework components implement module and validator contracts, proposal
|
Framework components implement module and validator contracts, proposal
|
||||||
|
|||||||
@@ -4,24 +4,39 @@ The [input contract](../integrations/transcript-glossary-files.md) owns input
|
|||||||
formats; [configuration](../config.md) owns settings; the
|
formats; [configuration](../config.md) owns settings; the
|
||||||
[output contract](../integrations/output-files.md) owns serialized results.
|
[output contract](../integrations/output-files.md) owns serialized results.
|
||||||
|
|
||||||
1. CLI filesystem adapters read transcript and glossary files. Schema parsing
|
1. CLI loads resolved configuration and transcript input. Preflight checks all
|
||||||
produces source objects; transcript segments enter canonical normalization.
|
selected glossary views and roles before the output session prepares resources.
|
||||||
2. Normalization sorts and merges segments using configured time and token
|
Transcript segments then enter canonical normalization.
|
||||||
limits, reassigning IDs where needed.
|
2. Normalization sorts segments by start time and preserves input order for
|
||||||
|
ties. It evaluates merge limits using the later end time and the same token
|
||||||
|
estimate as chunking, then reassigns IDs where needed.
|
||||||
3. The runner partitions the normalized transcript exactly once before its
|
3. The runner partitions the normalized transcript exactly once before its
|
||||||
module loop. Initial section identity, segment membership, and token
|
module loop. Initial section identity, segment membership, and token
|
||||||
estimates stay fixed while text evolves.
|
estimates stay fixed while text evolves.
|
||||||
4. Module instances execute in configured order. Sections may propose
|
4. Module instances execute in configured order. Each section hands its decoded
|
||||||
concurrently, but proposal indices follow section order and response order.
|
proposals and original local segment references to the runner along with
|
||||||
|
operation facts and any captured exchange. A bounded window admits sections
|
||||||
|
in order and lets validation and repair proceed as proposals become ready.
|
||||||
|
The runner assigns final proposal indices in section and response order
|
||||||
|
when it records completed work. It then writes diagnostics that use those
|
||||||
|
indices. PromptKit controls provider capacity across all three roles.
|
||||||
5. Deterministic validators precede LLM-backed validators. Every submitted
|
5. Deterministic validators precede LLM-backed validators. Every submitted
|
||||||
candidate receives one decision per validator on a normal return. Fatal returns
|
candidate receives one decision per validator on a normal return. Fatal returns
|
||||||
may preserve a checked subset of completed decisions as diagnostic evidence.
|
may preserve a checked subset of completed decisions as diagnostic evidence.
|
||||||
Recoverable failed batches mark their candidates unavailable; successful
|
Recoverable failed batches mark their candidates unavailable; successful
|
||||||
survivors continue through all required validators.
|
survivors continue through all required validators.
|
||||||
6. Only fully approved proposals reach application. Corrections apply in
|
6. Actionable rejections can enter independent section-local repair branches.
|
||||||
|
PromptKit coordinates each branch from the captured proposal exchange, owning
|
||||||
|
its history and additional-turn limit. Each revision, including the final
|
||||||
|
permitted response, is mapped to the original section and revalidated from the
|
||||||
|
start. Withdrawal, exhaustion, and recoverable failures finish the branch
|
||||||
|
without affecting independent approvals.
|
||||||
|
7. Only fully approved proposals reach application. Corrections apply in
|
||||||
proposal-index order within their owning section. Existing replacement
|
proposal-index order within their owning section. Existing replacement
|
||||||
policies and apply-time safety checks resolve stale conflicts.
|
policies and apply-time safety checks resolve stale conflicts. Skip
|
||||||
7. CLI encodes the final transcript and publishes active files through the
|
attribution uses available validation previews of the original text to
|
||||||
|
distinguish stale edits from defects in a proposal or repair response.
|
||||||
|
8. CLI encodes the final transcript and publishes active files through the
|
||||||
output session. `framework/processreport` builds one snapshot for YAML,
|
output session. `framework/processreport` builds one snapshot for YAML,
|
||||||
Markdown, and CLI totals. Debug evidence is captured separately when enabled.
|
Markdown, and CLI totals. Debug evidence is captured separately when enabled.
|
||||||
|
|
||||||
@@ -30,9 +45,15 @@ explicit `not_started`/`interrupted`/`completed` work units, compact model
|
|||||||
operation outcomes, and per-module results. Candidate payloads are retained
|
operation outcomes, and per-module results. Candidate payloads are retained
|
||||||
in the returned output only when debug capture is requested. A work unit can
|
in the returned output only when debug capture is requested. A work unit can
|
||||||
complete with model errors and applied corrections. Recoverable model failures
|
complete with model errors and applied corrections. Recoverable model failures
|
||||||
never trigger shared cancellation. Fatal setup, internal, application, or
|
never trigger shared cancellation. Fatal failures and parent cancellation stop
|
||||||
parent-cancellation errors cancel peers, join workers, and preserve earlier
|
further dispatch and join active workers. The initiating error carries its
|
||||||
committed module text for partial reports; no completed transcript is published.
|
processing phase and available origin. With ordinary reports enabled, work-unit
|
||||||
|
evidence includes available peer interruption details. The runner drains active
|
||||||
|
outcomes and assigns indices to usable proposals from later sections even when
|
||||||
|
it could not validate them. The working transcript advances at the module's
|
||||||
|
successful commit boundary.
|
||||||
|
Earlier committed text remains available to an interrupted run, and transcript
|
||||||
|
delivery requires successful processing.
|
||||||
|
|
||||||
Required transcript/report write failures are fatal. Optional debug writes
|
Required transcript/report write failures are fatal. Optional debug writes
|
||||||
report safely to stderr without changing decisions or exit status. See
|
report safely to stderr without changing decisions or exit status. See
|
||||||
|
|||||||
@@ -1,28 +1,54 @@
|
|||||||
# Audita Prompt Registry
|
# Audita Prompt Registry
|
||||||
|
|
||||||
`internal/prompts` embeds seven PromptKit YAML definitions under
|
`internal/prompts` embeds eleven PromptKit definitions in the `audita`
|
||||||
`assets/prompts`. The IDs and selected version remain:
|
collection. The four proposal IDs are `audita/modules.glossary.proposal`,
|
||||||
|
`audita/modules.homophones.proposal`, `audita/modules.spoken_word.proposal`,
|
||||||
|
and `audita/modules.grammar.proposal`; the corresponding repair IDs replace
|
||||||
|
`.proposal` with `.repair`. The three validator IDs are
|
||||||
|
`audita/validators.spoken_form_plausibility`,
|
||||||
|
`audita/validators.meaning_reversal_review`, and
|
||||||
|
`audita/validators.editorial_review`.
|
||||||
|
|
||||||
- `modules.glossary.proposal` (`v1`)
|
PromptKit composes embedded defaults with operator prompt sources. Shared
|
||||||
- `modules.homophones.proposal` (`v1`)
|
instructions come from the embedded fragments in `assets/shared`; the request
|
||||||
- `modules.spoken_word.proposal` (`v1`)
|
shared-variable helper fills absent `HardeningPolicy`, `ProposalContract`,
|
||||||
- `modules.grammar.proposal` (`v1`)
|
`RepairContract`, and `ValidationContract` variables. The processing adapter
|
||||||
- `validators.spoken_form_plausibility` (`v1`)
|
supplies the embedded instruction fragments for those names. Processing supplies
|
||||||
- `validators.meaning_reversal_review` (`v1`)
|
Audita's response-schema contract separately.
|
||||||
- `validators.editorial_review` (`v1`)
|
The operator's prompt definition controls which variables its template uses.
|
||||||
|
|
||||||
|
Proposal focus and validator acceptance intentionally differ: minor editorial
|
||||||
|
cleanup may be accepted even when the proposing module was not asked to perform
|
||||||
|
it. Follow the [architecture boundary](../policy/architecture.md#correction-and-concurrency-invariants)
|
||||||
|
when maintaining proposal, validation, and repair instructions; the difference
|
||||||
|
alone is not a reason to tighten validation. The testing rationale is recorded in
|
||||||
|
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md).
|
||||||
|
|
||||||
|
Bundled proposal system messages carry durable correction principles and the
|
||||||
|
editorial allowance during repair. The initial user message carries each
|
||||||
|
module's focused proposal assignment and exclusions. Repair can therefore give
|
||||||
|
a subsequent reassessment assignment without inheriting a system prohibition
|
||||||
|
on acceptable incidental cleanup. Review complete conversations when editing
|
||||||
|
these roles, including the transition from a correction set to one repair result.
|
||||||
|
|
||||||
|
For a standard request preview or validation, select a native PromptKit profile
|
||||||
|
and supply the template variables used by the selected prompt. The shipped
|
||||||
|
proposal definitions use `TranscriptDescriptionBlock`, `GlossaryJSON`,
|
||||||
|
`ProtectedTermsJSON`, and `SectionJSON`. Glossary entries provide context,
|
||||||
|
and the separate spelling list defines protection. Preview defaults the
|
||||||
|
spelling list to `[]` when omitted. The validator definitions use `TranscriptDescriptionBlock` and
|
||||||
|
`PayloadJSON`. The latter is supplied candidate JSON and does not require a
|
||||||
|
proposal inference call. Repair definitions use `RepairContract` and
|
||||||
|
`RepairRequestJSON`, containing the original proposal reference, current
|
||||||
|
correction, and ordered findings. When the target exists, the request also
|
||||||
|
includes `original_segment_text`; when the correction is applicable, it includes
|
||||||
|
`corrected_segment_text`. These fields describe a preview of the current version,
|
||||||
|
not an applied change. An inapplicable correction omits the resulting text and
|
||||||
|
retains its findings and available source text. An operator replacement may use
|
||||||
|
different variables.
|
||||||
|
See the [upstream CLI reference](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/cli.md)
|
||||||
|
for the standard preview, validation, and smoke commands.
|
||||||
|
|
||||||
PromptKit reads the embedded definitions and renders their system and user
|
|
||||||
messages from named string variables. The role-bound adapter supplies
|
|
||||||
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
|
|
||||||
embedded fragments under `assets/shared`, after caller variables so callers cannot replace those
|
|
||||||
instructions. Common literal-span and identity guidance lives in the proposal
|
|
||||||
fragment; shared candidate-reference, context, and confidence guidance lives in
|
|
||||||
the validation fragment. Module-specific policies and examples remain in YAML.
|
|
||||||
Proposal prompts receive the section JSON, glossary JSON where used, and transcript description block.
|
|
||||||
Validator prompts receive the correction payload and description block.
|
|
||||||
The prompt assets retain the prior task instructions and output-schema
|
|
||||||
association. The role-bound runtime supplies the complete JSON Schema output contract
|
|
||||||
and repair count; the definition does not own an operator-facing repair default.
|
|
||||||
The correction-set schema requires a string replacement and accepts an empty
|
The correction-set schema requires a string replacement and accepts an empty
|
||||||
string. Proposal guidance uses it for eligible span deletion and permits literal
|
string. Proposal guidance uses it for eligible span deletion and permits literal
|
||||||
spacing edits; validator guidance evaluates the full before/after segment under
|
spacing edits; validator guidance evaluates the full before/after segment under
|
||||||
@@ -30,26 +56,32 @@ each validator's existing remit. A blank resulting segment remains a domain
|
|||||||
rejection, not a structural schema error.
|
rejection, not a structural schema error.
|
||||||
|
|
||||||
`internal/framework/responseschema` remains the only embedded source for
|
`internal/framework/responseschema` remains the only embedded source for
|
||||||
`correction_set.v1.json` and `validator_decision_set.v1.json`. It exposes that
|
`correction_set.v1.json`, `correction_repair.v1.json`, and
|
||||||
|
`validator_decision_set.v1.json`. It exposes that
|
||||||
filesystem to PromptKit and retains Audita's schema-key lookup. PromptKit's
|
filesystem to PromptKit and retains Audita's schema-key lookup. PromptKit's
|
||||||
prepared execution supplies definition and rendered hashes; Audita does not
|
prepared execution supplies definition and rendered hashes; Audita does not
|
||||||
recompute or promise equality with hashes from the removed template renderer.
|
recompute or promise equality with hashes from the removed template renderer.
|
||||||
The small Audita registry records only prompt ID, version, source, and embedded
|
The small Audita registry records only prompt ID, source, and embedded path. Prepared runtime metadata belongs in diagnostics rather than the registry.
|
||||||
path. Prepared runtime metadata belongs in diagnostics rather than the registry.
|
|
||||||
|
|
||||||
The structured-call contract carries prompt ID/version, string variables,
|
The structured-call contract carries prompt ID, string variables,
|
||||||
response-schema key, and logical-call attribution. Module and validator callers
|
response-schema key, and logical-call attribution. Module and validator callers
|
||||||
pass these values through the role-bound adapter. See the
|
pass these values through the role-bound adapter. See the
|
||||||
[LLM runtime](llm-runtime.md) for runtime assembly and
|
[LLM runtime](llm-runtime.md) for runtime assembly and
|
||||||
[configuration](../config.md) for operator-selected profiles.
|
[configuration](../config.md) for operator-selected profiles.
|
||||||
|
|
||||||
Proposal instructions describe literal matching, each module's replacement policy,
|
Proposal instructions describe literal matching, each module's replacement policy,
|
||||||
source-based proposal identity, and confidence in whether an edit is warranted.
|
and source-based proposal identity.
|
||||||
|
They request independently useful edits and a combined proposal when edits
|
||||||
|
overlap or depend on one another, while allowing independent edits to share a
|
||||||
|
segment. This is generation guidance; application still uses the existing
|
||||||
|
ordering and applicability checks.
|
||||||
Proposal `id` and validator `correction_index` values are short, one-based
|
Proposal `id` and validator `correction_index` values are short, one-based
|
||||||
references scoped to each request, not internal transcript/proposal identities.
|
references scoped to each request, not internal transcript/proposal identities.
|
||||||
Mapping belongs to [proposal generation](modules.md) and
|
Mapping belongs to [proposal generation](modules.md) and
|
||||||
[validator reconciliation](validators.md). Validator instructions distinguish
|
[validator reconciliation](validators.md). Validator decisions use `approved` for the verdict. Approval examples use an empty
|
||||||
confidence in the verdict from the verdict itself. Compact output examples
|
`reason`; rejection guidance requests a specific defect and supplied evidence
|
||||||
|
within the validator's remit. The model explains defects; repair owns revision or
|
||||||
|
withdrawal. This is prompt guidance, not a runtime nonempty-reason check. Compact output examples
|
||||||
illustrate the supplied schema without requiring fixed JSON formatting or key
|
illustrate the supplied schema without requiring fixed JSON formatting or key
|
||||||
order. The grammar article exception and glossary mishearing exception apply
|
order. The grammar article exception and glossary mishearing exception apply
|
||||||
consistently across their instructions.
|
consistently across their instructions.
|
||||||
@@ -57,3 +89,35 @@ consistently across their instructions.
|
|||||||
Only the three active validator definitions are retained. The unused internal
|
Only the three active validator definitions are retained. The unused internal
|
||||||
`grammar_review` and `spoken_word_review` definitions and type selections were
|
`grammar_review` and `spoken_word_review` definitions and type selections were
|
||||||
removed; built-in chains and operator configuration are unchanged.
|
removed; built-in chains and operator configuration are unchanged.
|
||||||
|
|
||||||
|
Repair guidance reassesses the proposal against the original transcript, context,
|
||||||
|
and findings. Unsupported, unnecessary, or unresolved uncertain corrections may
|
||||||
|
be withdrawn, preserving the original text. Otherwise acceptable minor editorial
|
||||||
|
cleanup remains permissible during repair.
|
||||||
|
|
||||||
|
Repair responses keep `proposal_ref` at the root and nest `action` and
|
||||||
|
`correction` inside `decision`. The schema requires a correction object for
|
||||||
|
`revise` and `null` for `withdraw`, using a nested union compatible with strict
|
||||||
|
structured outputs. Mismatched combinations enter PromptKit's configured
|
||||||
|
structural-repair path before domain decoding. Audita retains its domain check
|
||||||
|
and verifies the proposal reference. Operator replacement prompts must return
|
||||||
|
this nested response format; repair request inputs retain their existing shape.
|
||||||
|
|
||||||
|
Glossary repair distinguishes the supplied protected spellings from eligible
|
||||||
|
glossary correction targets; suspected mishearing does not override that protected
|
||||||
|
list.
|
||||||
|
|
||||||
|
Examples contrast meaningful apparent filler with removable hesitation and show
|
||||||
|
both revision and withdrawal. Treat their words as contextual illustrations,
|
||||||
|
not unconditional triggers for deletion or replacement.
|
||||||
|
|
||||||
|
The meaning-reversal validator has a deliberately narrow remit: reversal of the
|
||||||
|
full segment's assertion, not every change of meaning. Its prompt distinguishes
|
||||||
|
changes of referent from reversal and asks rejection reasons to identify the
|
||||||
|
opposing assertions. Preserve its carefully tuned core instructions when editing;
|
||||||
|
other validators assess other criteria.
|
||||||
|
|
||||||
|
Prompt tests protect rendering with supplied data, resource selection, schemas,
|
||||||
|
and integration behavior. They do not freeze bundled prose with word/phrase
|
||||||
|
checks or snapshots. Review editorial intent directly; offline tests cannot
|
||||||
|
establish live-model editorial quality.
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ Execution-class metadata:
|
|||||||
|
|
||||||
Deterministic:
|
Deterministic:
|
||||||
- `proposal_shape`
|
- `proposal_shape`
|
||||||
- `confidence_threshold`
|
|
||||||
- `original_text_presence`
|
- `original_text_presence`
|
||||||
- `non_empty_resulting_segment` — rejects edits that would leave the segment
|
- `non_empty_resulting_segment` — rejects edits that would leave the segment
|
||||||
blank, regardless of replacement-span length.
|
blank, regardless of replacement-span length.
|
||||||
@@ -46,9 +45,25 @@ For each module section:
|
|||||||
3. record decisions and compact operation outcomes;
|
3. record decisions and compact operation outcomes;
|
||||||
4. carry only approved proposals forward.
|
4. carry only approved proposals forward.
|
||||||
|
|
||||||
Decision cardinality is enforced inside Audita: each candidate proposal must
|
When correction repair is enabled, the runner also gathers independent findings
|
||||||
receive exactly one outcome per validator it reaches. The runner still rejects
|
for rejected candidates when a validator has usable evidence. Unavailable
|
||||||
duplicate, missing, or unknown identities in an internal validator result.
|
assessments remain unavailable, and only candidates that pass every applicable
|
||||||
|
check can advance. A known rejection can enter repair even when another
|
||||||
|
assessment is unavailable; an unavailable assessment alone does not. Revised
|
||||||
|
candidates run the applicable checks again. A zero repair budget retains the
|
||||||
|
original shortlist flow.
|
||||||
|
|
||||||
|
The shared validator port lives in `internal/framework/validators`. Instances
|
||||||
|
may run concurrently across sections and can keep synchronized state. Request
|
||||||
|
data is borrowed for the duration of the call. Copy data before modifying or
|
||||||
|
retaining it.
|
||||||
|
|
||||||
|
The runner checks decision identity and disposition before allowing a candidate
|
||||||
|
to advance. A normal return supplies one decision per submitted candidate. A
|
||||||
|
fatal return may provide a checked subset as evidence. Contradictory approval
|
||||||
|
and disposition are a fatal contract failure. The recorded disposition governs
|
||||||
|
approval, and each record carries the effective execution class, including
|
||||||
|
instance overrides, for initial validation and revalidation.
|
||||||
|
|
||||||
## LLM Validator Batching
|
## LLM Validator Batching
|
||||||
|
|
||||||
@@ -72,9 +87,7 @@ makes all requested candidates unavailable. Decisions and unavailable indices
|
|||||||
map back to internal proposal identities; results are ordered by request identity
|
map back to internal proposal identities; results are ordered by request identity
|
||||||
and satisfy strict internal cardinality before the runner advances
|
and satisfy strict internal cardinality before the runner advances
|
||||||
approved candidates. An unavailable candidate never advances as approved.
|
approved candidates. An unavailable candidate never advances as approved.
|
||||||
Invalid confidence in a decoded test double also makes only its candidate
|
A partially unavailable batch records one failed
|
||||||
unavailable; production schema validation still rejects out-of-range confidence
|
|
||||||
before reconciliation. A partially unavailable batch records one failed
|
|
||||||
operation with only those requested indices, while a successful batch with
|
operation with only those requested indices, while a successful batch with
|
||||||
ignored extras records a decision-filtering recovery. Fatal errors stop the
|
ignored extras records a decision-filtering recovery. Fatal errors stop the
|
||||||
chain without returning unchecked candidates as approved. A local result builder
|
chain without returning unchecked candidates as approved. A local result builder
|
||||||
@@ -84,8 +97,11 @@ Reconciliation makes no semantic retry or extra model call; guided recovery rema
|
|||||||
[deferred](../roadmap/future.md#domain-aware-response-recovery).
|
[deferred](../roadmap/future.md#domain-aware-response-recovery).
|
||||||
|
|
||||||
Debug request metadata records `proposal_indices` in local-reference order
|
Debug request metadata records `proposal_indices` in local-reference order
|
||||||
(reference 1 is the first entry). Raw decoded responses and ignored indices use
|
(reference 1 is the first entry). Decoding preserves the supplied numeric
|
||||||
local references; reconciled decisions and unavailable indices use internal
|
tokens, including integral decimal or exponent spellings. Reconciliation
|
||||||
|
checks the batch range before converting to a local integer, so an extra
|
||||||
|
reference beyond the native integer range can be ignored. Raw decoded responses
|
||||||
|
and ignored indices use local references. Reconciled decisions use internal
|
||||||
proposal indices. Batch token estimates use the same locally numbered payload
|
proposal indices. Batch token estimates use the same locally numbered payload
|
||||||
as prompt encoding.
|
as prompt encoding.
|
||||||
|
|
||||||
@@ -116,19 +132,23 @@ is covered by [report assembly](diagnostics-reporting.md).
|
|||||||
- `internal/validators/*_test.go`
|
- `internal/validators/*_test.go`
|
||||||
- `internal/framework/validators/*_test.go`
|
- `internal/framework/validators/*_test.go`
|
||||||
- `internal/framework/processreport/correction_ledger_test.go`
|
- `internal/framework/processreport/correction_ledger_test.go`
|
||||||
- `internal/cli/run_test.go`
|
- `internal/cli/process_ordered_integration_test.go`
|
||||||
|
|
||||||
## Supplemental Validation Context
|
## Supplemental Validation Context
|
||||||
|
|
||||||
Request construction prepares one segment-position lookup and protected vocabulary
|
Each stage prepares a context matcher and indexes its selected glossary entries
|
||||||
for all candidates. Glossary selection remains candidate-specific, including the
|
by identity. Request construction uses a segment-position lookup and returns
|
||||||
proposed replacement text.
|
matched entries in selection order for each candidate, including matches in the
|
||||||
|
replacement text. The deterministic guard and application check share the
|
||||||
|
stage's separate protection matcher.
|
||||||
|
|
||||||
Each candidate includes the immediately preceding and following source segments
|
Each candidate carries its target segment's supplied `speaker`, including an
|
||||||
available within its fixed chunk, labeled `preceding` or `following` with speaker,
|
empty value when no speaker is provided. It also includes the immediately
|
||||||
|
preceding and following source segments available within its fixed chunk,
|
||||||
|
labeled `preceding` or `following` with speaker,
|
||||||
text, and categories, plus glossary entries whose names, aliases, or
|
text, and categories, plus glossary entries whose names, aliases, or
|
||||||
recognized plurals appear in its original/resulting text or those neighbors.
|
recognized plurals appear in its original/resulting text or those neighbors.
|
||||||
Glossary matching reuses the protected vocabulary; entries retain glossary order.
|
Glossary matching uses the same recognition rules; entries retain selection order.
|
||||||
Model-facing candidate and neighboring context omit internal segment IDs.
|
Model-facing candidate and neighboring context omit internal segment IDs.
|
||||||
This supplies evidence such as a troupe definition for a proposed `Jesters`
|
This supplies evidence such as a troupe definition for a proposed `Jesters`
|
||||||
correction or an adjacent speaker turn explaining deliberate repetition. A correction
|
correction or an adjacent speaker turn explaining deliberate repetition. A correction
|
||||||
@@ -136,15 +156,26 @@ already clear from its own segment does not require such evidence. No audio,
|
|||||||
out-of-chunk context, or complete transcript is implied.
|
out-of-chunk context, or complete transcript is implied.
|
||||||
|
|
||||||
Context is serialized inside the validation item and included in the existing
|
Context is serialized inside the validation item and included in the existing
|
||||||
payload token estimate. If one enriched item exceeds the configured limit, optional
|
payload token estimate, including the target speaker. Fitting measures the
|
||||||
neighbors are removed from the end first, then glossary entries from the end,
|
required data first, then considers glossary entries followed by neighboring
|
||||||
until it fits or only required candidate data remains. Required before/after text
|
segments in order. It stops at the first entry that exceeds the budget. Required
|
||||||
is never truncated. Existing oversized-candidate handling still applies if that
|
before/after text is never truncated. A candidate whose required data exceeds
|
||||||
required data does not fit. Context is supplemental, not a promise that every
|
the budget receives an unavailable decision. The built-in estimator reuses
|
||||||
relevant definition or neighboring utterance always fits. This uses the existing
|
encoded fragment counts for fitting and batching.
|
||||||
payload estimator, not a new guarantee about the provider's full context window.
|
|
||||||
|
|
||||||
## Protected-Name Repetition
|
## Protected Names
|
||||||
|
|
||||||
|
Protection compares the source segment with the full replacement preview.
|
||||||
|
Missing and ambiguous spans have no protection verdict. An empty preview can
|
||||||
|
produce a protection finding alongside the blank-result rule. Application
|
||||||
|
rechecks the preview against the current segment after earlier edits.
|
||||||
|
|
||||||
|
The glossary matcher selects the longest spelling at a match position.
|
||||||
|
Punctuation-edged spellings use literal edge checks with Go's ASCII regexp
|
||||||
|
boundary semantics. Active glossary exemptions take part in match selection.
|
||||||
|
Validation context uses the same scan to select identities in glossary order.
|
||||||
|
|
||||||
|
### Protected-Name Repetition
|
||||||
|
|
||||||
Spoken-word cleanup may reduce repeated protected-name occurrences when every
|
Spoken-word cleanup may reduce repeated protected-name occurrences when every
|
||||||
original protected identity still appears in the resulting segment and surviving
|
original protected identity still appears in the resulting segment and surviving
|
||||||
@@ -153,4 +184,4 @@ both a replacement of the repeated phrase and deletion of a redundant span. This
|
|||||||
exception only relaxes the deterministic count guard; editorial and meaning-reversal
|
exception only relaxes the deterministic count guard; editorial and meaning-reversal
|
||||||
validation still decide whether the repetition was accidental and the edit preserves
|
validation still decide whether the repetition was accidental and the edit preserves
|
||||||
meaning. Intentional emphasis and substantive references remain protected by those
|
meaning. Intentional emphasis and substantive references remain protected by those
|
||||||
reviews. Other modules retain their existing protection rules.
|
reviews.
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
# Migrating to Audita v2
|
# Migrating to Audita v2
|
||||||
|
|
||||||
|
This guide describes the historical v2 release. For current commands,
|
||||||
|
configuration, and example projects, use the [v3 migration guide](v3.md).
|
||||||
|
|
||||||
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
|
||||||
Version-1 configuration files, environment overlays, and retired flags are not
|
Version-1 configuration files, environment overlays, and retired flags are not
|
||||||
accepted. Copy a maintained [local](../../examples/minimal-config.yml) or
|
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or
|
||||||
[service](../../examples/production-config.yml) example, then select real models
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/production-config.yml) example, then select real models
|
||||||
in the accompanying `examples/profiles/` files. A testing overlay is shown at
|
in the accompanying `examples/profiles/` files. A testing overlay is shown at
|
||||||
`examples/overlays/testing.yml`; its model choice carries no cost, speed, or
|
`examples/overlays/testing.yml`; its model choice carries no cost, speed, or
|
||||||
quality guarantee.
|
quality guarantee.
|
||||||
@@ -121,8 +124,7 @@ provider attempts. With debug enabled, utilization remains grouped by backend
|
|||||||
capacity and usage fields retain known/complete flags. Debug runs write
|
capacity and usage fields retain known/complete flags. Debug runs write
|
||||||
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
|
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
|
||||||
`config-origins.yaml`, `utilization-diagnostics.json`, and
|
`config-origins.yaml`, `utilization-diagnostics.json`, and
|
||||||
`llm-execution.json`. Request and response artifacts remain redacted debugging
|
`llm-execution.json`. Request and response artifacts are debugging material. See [operations](../operations.md#debug-evidence) for placement
|
||||||
material. See [operations](../operations.md#diagnostics-directory) for placement
|
|
||||||
and paths, [configuration](../config.md) for all settings, and the
|
and paths, [configuration](../config.md) for all settings, and the
|
||||||
[CLI reference](../cli.md) for current flags.
|
[CLI reference](../cli.md) for current flags.
|
||||||
|
|
||||||
|
|||||||
41
docs/migrations/v3.md
Normal file
41
docs/migrations/v3.md
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
# Migrating to application configuration v3
|
||||||
|
|
||||||
|
Audita v4.0.0 adopts application configuration schema v3. Audita v3.1.x uses
|
||||||
|
schema v2. Set `version: 3` in the application file and move glossary entries into
|
||||||
|
configuration. Use the [configuration reference](../config.md) and
|
||||||
|
[maintained examples](../../examples/) for the new formats.
|
||||||
|
|
||||||
|
- Replace `pipeline.modules` with ordered `pipeline.stages` records containing
|
||||||
|
`module` and any per-entry overrides. An explicit empty list runs normalization.
|
||||||
|
- Define named `glossary_collections` in configuration or its companion imports;
|
||||||
|
select them using `glossaries` and `protected_terms`. Rename each legacy
|
||||||
|
entry's `summary` to `description` to preserve its prompt context. Remove
|
||||||
|
`--glossary` and `--modules` from invocations.
|
||||||
|
- Move profiles and backends out of the application configuration and retired
|
||||||
|
`profiles_dir` layout into PromptKit resources. Use its current list records,
|
||||||
|
IDs, credential sources, and backend admission settings. Select proposal and
|
||||||
|
validation profiles in application `llm` settings.
|
||||||
|
- Use the shared `--debug` flag for capture; `diagnostics.work_dir` remains the
|
||||||
|
parent directory. Review report and output destinations before running.
|
||||||
|
- Completion counts are hidden by default. Use `--verbose` to see them on
|
||||||
|
stderr, including when ordinary reports are disabled. `--debug` also shows
|
||||||
|
counts unless followed by `--quiet`. See [CLI reporting](../cli.md#processing).
|
||||||
|
|
||||||
|
## Confidence removal
|
||||||
|
|
||||||
|
Remove root `thresholds` and stage `confidence_threshold` settings, and the
|
||||||
|
`--glossary-confidence-threshold`, `--homophones-confidence-threshold`,
|
||||||
|
`--spoken-word-confidence-threshold`, and `--grammar-confidence-threshold` flags.
|
||||||
|
The `confidence_threshold` validator has been removed from the built-in chains.
|
||||||
|
|
||||||
|
Update custom proposal, validation, and repair prompts to omit `confidence` from
|
||||||
|
response examples and instructions. Audita's response schemas no longer accept
|
||||||
|
that field. Validator responses still require `approved` and `reason`; approvals
|
||||||
|
may use an empty reason. Corrections in debug records and other affected
|
||||||
|
serialized proposal/repair data no longer carry confidence, so consumers must
|
||||||
|
stop requiring or interpreting it. No replacement score or threshold is provided.
|
||||||
|
|
||||||
|
Inspect configuration and resources before processing, then try a controlled
|
||||||
|
small transcript. Processing preflight checks every glossary correction and
|
||||||
|
selected role before preparing outputs or invoking a model. See [operations](../operations.md)
|
||||||
|
for lifecycle and recovery.
|
||||||
@@ -1,83 +1,91 @@
|
|||||||
# Audita Operations
|
# Audita Operations
|
||||||
|
|
||||||
This guide owns physical output placement and failure handling. The
|
This guide owns output placement and failure handling. The [CLI reference](cli.md)
|
||||||
[CLI reference](cli.md) owns flags and streams, [configuration](config.md) owns
|
owns invocation, [configuration](config.md) owns settings, and the
|
||||||
settings, and the [output contract](integrations/output-files.md) owns schemas.
|
[output contract](integrations/output-files.md) owns serialized formats.
|
||||||
|
|
||||||
## Normal Workflow
|
## Processing
|
||||||
|
|
||||||
1. Validate a version-2 configuration and its selected profiles with
|
Inspect the application and PromptKit resources, prepare a transcript, then run
|
||||||
`audita config validate --config config.yml`.
|
`audita process` with the desired destination. The [examples](../examples/README.md)
|
||||||
2. Prepare a transcript and glossary following the [input contract](integrations/transcript-glossary-files.md).
|
show complete projects. Processing preflight checks every configured glossary
|
||||||
3. Run `audita process input.json --config config.yml --glossary glossary.yaml --output corrected.json`.
|
correction and selected profile before output preparation and model work. Empty
|
||||||
4. Check the exit status, stderr summary, corrected transcript, and reports.
|
protection and an empty pipeline are valid. A completed run can contain
|
||||||
A completed run can contain recoverable model errors and still apply
|
recoverable model errors while still applying independently validated edits.
|
||||||
corrections that passed every required validator.
|
Correction repair sends the complete original proposal exchange and branch
|
||||||
|
history to the selected repair provider. A provider rejection ends that branch
|
||||||
|
as unavailable; other approved edits can still be applied. See
|
||||||
|
[configuration](config.md#settings) for the attempt budget and profile selection.
|
||||||
|
|
||||||
## Required Output Files
|
## Configuration trust
|
||||||
|
|
||||||
Reporting is enabled by default. For file transcript output, `errors.yml`,
|
Treat the selected main file, imports, overlay, and automatically loaded
|
||||||
`recoveries.yml`, and `report.md` go in the transcript output's parent directory.
|
glossary and prompt companions as one trusted bundle. Choose the working
|
||||||
For stdout transcript output, they go in `reporting.stdout_dir` (default `.`),
|
directory deliberately when using automatic discovery. A bundle can select
|
||||||
overridden by `--report-dir`. `--reporting=false` suppresses all three reports
|
the provider, prompt content, and local files consumed by processing. Review
|
||||||
and removes their path requirements. It does not suppress the stderr summary or
|
these sources before running with sensitive transcript data. Native inspection
|
||||||
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
|
describes the loaded configuration and resources. Processing readiness also
|
||||||
|
depends on the profiles and glossaries selected by the pipeline. See
|
||||||
|
[configuration loading](config.md#loading-and-composition) for discovery rules.
|
||||||
|
|
||||||
Existing report directories and parents for regular transcript files must be
|
## Outputs and lifecycle
|
||||||
writable. Audita prevents report writes from overwriting transcript, glossary,
|
|
||||||
loaded configuration, or loaded profile sources, and rejects conflicting output
|
|
||||||
destinations. The final transcript may intentionally replace its own input after
|
|
||||||
successful processing; other input/configuration/profile sources remain protected.
|
|
||||||
Preparation never truncates an existing transcript. Device and pipe destinations
|
|
||||||
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
|
|
||||||
they are not opened during preflight or replaced with regular files. A pipe write
|
|
||||||
can wait for its reader, and stream write errors are fatal. With reporting enabled,
|
|
||||||
reports still use the output path's parent; disable reporting when that placement
|
|
||||||
is unsuitable for a stream destination.
|
|
||||||
|
|
||||||
With reporting enabled, all three reports are initialized as `in_progress` before
|
Reporting is enabled by default. For transcript file output, `errors.yml`,
|
||||||
input processing. Output is encoded fully before publication. Each regular output
|
`recoveries.yml`, and `report.md` are placed beside that file unless `--report-dir`
|
||||||
file is replaced through a temporary file on its destination filesystem:
|
is supplied. For stdout transcript output they use the effective report directory.
|
||||||
transcript first, then `errors.yml`,
|
Disabling reporting leaves its destination inactive and unprobed. Quiet mode
|
||||||
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
|
requires a transcript file or `/dev/null`; it suppresses optional messages, not
|
||||||
multi-file transaction. New files honor the process umask; replacing an existing
|
failures. Stdout carries transcript JSON when selected.
|
||||||
file preserves its permission bits. A late report error may leave the transcript
|
|
||||||
published; the process exits nonzero and attempts failed-status reports. On fatal processing
|
|
||||||
or cancellation, no completed transcript is published, but partial reports are
|
|
||||||
finalized where possible. An uncatchable termination can leave `in_progress`.
|
|
||||||
|
|
||||||
## Diagnostics Directory
|
Before processing, Audita checks destination collisions against the transcript,
|
||||||
|
consumed application and PromptKit sources, and effective host message files. An
|
||||||
|
intentional in-place transcript replacement is allowed. A regular transcript
|
||||||
|
output is prepared in a sibling temporary file, leaving the final destination
|
||||||
|
untouched until successful publication. Pipes and devices are opened only for
|
||||||
|
delivery, and a pipe can wait for its reader. If a transcript destination changes
|
||||||
|
between a regular file and a pipe or device during processing, Audita can report a
|
||||||
|
delivery failure while preserving the new destination. Parent directories for
|
||||||
|
required outputs must already exist. Publication uses the destinations resolved
|
||||||
|
during preparation. Give concurrent invocations separate destinations, and
|
||||||
|
choose separate report directories when retaining reports from multiple runs.
|
||||||
|
The caller owns these directories and their retention. Later runs can replace
|
||||||
|
ordinary report files at the same destination.
|
||||||
|
|
||||||
`--debug` enables detailed evidence independently of reporting. One directory
|
Enabled reports are initialized as `in_progress` before processing. The
|
||||||
named `run-<id>` is created under `diagnostics.work_dir` (default `/tmp/audita`)
|
transcript is published before final reports; the files do not form a single
|
||||||
and retained on success or failure. Without debug, no directory or payload
|
transaction. A late report failure may therefore leave a published transcript.
|
||||||
files are created. The parent is created when needed only for debug. Failure to
|
After a required write failure or cancellation, Audita attempts to clean
|
||||||
write optional debug evidence prints a redacted stderr diagnostic but does not
|
unpublished temporary resources and publish failed-status reports. A cleanup
|
||||||
change correction decisions or the exit status.
|
failure is reported with the earlier error. Cancellation observed after a
|
||||||
|
stream write still fails the invocation, even when some or all bytes reached
|
||||||
|
the reader. If failed reports cannot be rendered, available report evidence
|
||||||
|
remains. An uncatchable termination can leave `in_progress` evidence. Use the
|
||||||
|
exit status and report status to determine the outcome.
|
||||||
|
|
||||||
Debug evidence includes invocation and effective settings, configuration
|
## Live progress
|
||||||
origins, selected profiles, redacted source and normalized snapshots,
|
|
||||||
normalization and initial chunk summaries, all captured proposed candidates
|
|
||||||
and validator decisions in `correction-ledger.json`, execution/utilization
|
|
||||||
records, and model interaction files. Individual artifact writes can fail; a
|
|
||||||
directory's existence does not prove every file was written. These files can
|
|
||||||
contain transcript text and prompt/response payloads. Configured backend
|
|
||||||
credentials are redacted, but the transcript itself is not anonymized. Protect
|
|
||||||
and retain the directory according to the sensitivity of the input.
|
|
||||||
|
|
||||||
## Failure Inspection and Recovery
|
Use `--progress` to follow stage, chunk, proposal, validation, and repair
|
||||||
|
activity on stderr. Report and transcript files provide the retained outcome.
|
||||||
|
Debug mode enables live progress by default. Use `--progress=false` to keep
|
||||||
|
debug evidence with progress disabled.
|
||||||
|
|
||||||
Exit `0` means processing and every enabled required output completed. Exit
|
## Debug evidence
|
||||||
`1` means cancellation, fatal processing, or required-output failure. Exit `2`
|
|
||||||
identifies usage or configuration/destination selection errors. Recoverable
|
|
||||||
model-operation failures appear in `errors.yml` as one record per affected
|
|
||||||
work unit; they do not automatically make the run fail. `recoveries.yml`
|
|
||||||
records successful operations that needed retry, repair, or proposal filtering.
|
|
||||||
`report.md` shows every initial chunk and configured module instance, including
|
|
||||||
interrupted and unattempted work. Ordinary reports omit transcript excerpts and
|
|
||||||
full decision ledgers; enable debug when those are needed.
|
|
||||||
|
|
||||||
Read stderr and report status first. If a required output failed after a
|
The shared `--debug` flag enables retained artifacts in a run directory beneath
|
||||||
transcript was published, do not treat the file alone as proof of a completed
|
`diagnostics.work_dir`; selecting a work directory alone does not enable capture.
|
||||||
run. Fix the input, service, or destination problem and rerun. Audita has no
|
Optional debug preparation or writes can produce warnings without
|
||||||
resume/checkpoint API.
|
changing correction decisions or the required output result. Debug material can
|
||||||
|
include transcript content, prompts, responses, and correction decisions.
|
||||||
|
Protect this material according to the sensitivity of the inputs and responses
|
||||||
|
and manage its retention. Repair interactions
|
||||||
|
and revalidation artifacts include proposal and attempt coordinates; the
|
||||||
|
correction ledger connects them to the original edit.
|
||||||
|
|
||||||
|
## Recovery
|
||||||
|
|
||||||
|
Exit `0` indicates completion of processing and required outputs; nonzero status
|
||||||
|
indicates an input, processing, cancellation, or required-output failure.
|
||||||
|
Recoverable model-operation failures and successful repairs or retries are
|
||||||
|
recorded separately in the ordinary YAML reports. The Markdown matrices identify
|
||||||
|
pipeline position, module, and optional operator ID. Fix the input, service, or
|
||||||
|
destination and rerun when necessary; Audita has no resume or checkpoint API.
|
||||||
|
|||||||
@@ -2,7 +2,12 @@
|
|||||||
|
|
||||||
## Standard Repository Policies
|
## Standard Repository Policies
|
||||||
|
|
||||||
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy Reading Guide in `docs/development.md`, adapted for this repository.
|
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy
|
||||||
|
Reading Guide in `docs/development.md`, adapted for this repository.
|
||||||
|
`docs/policy/documentation.md`, `docs/policy/oss.md`, and `docs/policy/release.md`
|
||||||
|
incorporate revision `593c9b52273c5c879785ae9e995d21026df467b4`.
|
||||||
|
The remaining listed material incorporates revision
|
||||||
|
`d84a38ebab26f048cafbe59a2246700c35d29843`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -10,13 +15,17 @@ SPDX-License-Identifier: CC-BY-SA-4.0
|
|||||||
|
|
||||||
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/COPYRIGHT) |
|
||||||
|
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
|
||||||
|
|
||||||
|
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4).
|
||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.4.2`.
|
Selected module: `go.maximumdirect.net/configkit` at `v1.2.1`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -24,83 +33,145 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f/COPYRIGHT).
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/51ef3144158db1a09fabdbd1c7ae59908c84923b/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/67c03690181c5af8dde6be325abac241dae6732f).
|
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/51ef3144158db1a09fabdbd1c7ae59908c84923b).
|
||||||
|
|
||||||
|
## ReleaseKit
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/releasekit` at `v0.10.0`.
|
||||||
|
|
||||||
|
Material: Adapted launcher and tag-delivery recipes in `scripts/release.sh`
|
||||||
|
and `.woodpecker/release.yml`, attributed to ReleaseKit `v0.9.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243/COPYRIGHT) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/82bac1a3ab935beddd4b8789d1e07f52135d4243).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4).
|
||||||
|
|
||||||
|
## CommandKit
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/commandkit` at `v0.7.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/b57bfe176cff05a892e6cb07ebb5f350239e55d4/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/b57bfe176cff05a892e6cb07ebb5f350239e55d4).
|
||||||
|
|
||||||
|
## CommandKit ckconfig
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/ckconfig` at `v0.9.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/9962368f0c1ea1f6ad010638e94621091105d745/integrations/ckconfig/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/9962368f0c1ea1f6ad010638e94621091105d745).
|
||||||
|
|
||||||
|
## CommandKit ckprompt
|
||||||
|
|
||||||
|
Selected module: `go.maximumdirect.net/ckprompt` at `v0.10.0`.
|
||||||
|
|
||||||
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/c9a56e1b15f2169881a91fca4e1f4a1c475e057a/integrations/ckprompt/COPYRIGHT).
|
||||||
|
|
||||||
|
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/c9a56e1b15f2169881a91fca4e1f4a1c475e057a).
|
||||||
|
|
||||||
## Go runtime and standard library
|
## Go runtime and standard library
|
||||||
|
|
||||||
Selected toolchain: Go 1.25.5 for the distributed Linux binaries.
|
Selected toolchain: Go 1.27.4 for the distributed Linux binaries.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
|
[Source](https://github.com/golang/go/commit/7b2881024c95a11ddb86663a616340aa3a88b67f).
|
||||||
|
|
||||||
## Go vendored golang.org/x/crypto
|
## Go vendored golang.org/x/crypto
|
||||||
|
|
||||||
Material: `golang.org/x/crypto` at `v0.39.0` in the Go 1.25.5 standard library.
|
Material: `golang.org/x/crypto` at `v0.52.1-0.20260526024921-9beb694f9766` in the Go 1.27.4 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/crypto/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/crypto/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/crypto/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/crypto/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
|
[Source](https://github.com/golang/go/commit/7b2881024c95a11ddb86663a616340aa3a88b67f).
|
||||||
|
|
||||||
## Go vendored golang.org/x/net
|
## Go vendored golang.org/x/net
|
||||||
|
|
||||||
Material: `golang.org/x/net` at `v0.41.0` in the Go 1.25.5 standard library.
|
Material: `golang.org/x/net` at `v0.55.1-0.20260731170536-c1d18010be90` in the Go 1.27.4 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/net/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/net/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/net/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/net/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
|
[Source](https://github.com/golang/go/commit/7b2881024c95a11ddb86663a616340aa3a88b67f).
|
||||||
|
|
||||||
## Go vendored golang.org/x/sys
|
## Go vendored golang.org/x/sys
|
||||||
|
|
||||||
Material: `golang.org/x/sys` at `v0.33.0` in the Go 1.25.5 standard library.
|
Material: `golang.org/x/sys` at `v0.45.0` in the Go 1.27.4 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/sys/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/sys/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/sys/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/sys/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
|
[Source](https://github.com/golang/go/commit/7b2881024c95a11ddb86663a616340aa3a88b67f).
|
||||||
|
|
||||||
## Go vendored golang.org/x/text
|
## Go vendored golang.org/x/text
|
||||||
|
|
||||||
Material: `golang.org/x/text` at `v0.26.0` in the Go 1.25.5 standard library.
|
Material: `golang.org/x/text` at `v0.37.0` in the Go 1.27.4 standard library.
|
||||||
|
|
||||||
Copyright 2009 The Go Authors.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/text/LICENSE) |
|
[LICENSE](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/text/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/text/PATENTS).
|
[PATENTS](https://github.com/golang/go/blob/7b2881024c95a11ddb86663a616340aa3a88b67f/src/vendor/golang.org/x/text/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
|
[Source](https://github.com/golang/go/commit/7b2881024c95a11ddb86663a616340aa3a88b67f).
|
||||||
|
|
||||||
## go-yaml v4
|
## go-yaml v4
|
||||||
|
|
||||||
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0-rc.6`.
|
Selected module: `go.yaml.in/yaml/v4` at `v4.0.0`.
|
||||||
|
|
||||||
Copyright 2006-2010 Kirill Simonov
|
Copyright 2006-2010 Kirill Simonov
|
||||||
|
|
||||||
@@ -112,37 +183,37 @@ SPDX-License-Identifier: Apache-2.0 AND MIT
|
|||||||
|
|
||||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||||
[MIT](licenses/MIT-libyaml-v4.txt) |
|
[MIT](licenses/MIT-libyaml-v4.txt) |
|
||||||
[NOTICE](https://github.com/yaml/go-yaml/blob/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125/NOTICE).
|
[NOTICE](https://github.com/yaml/go-yaml/blob/f2f9563503f11b7a60da0c532175dcc693ce8d5a/NOTICE).
|
||||||
|
|
||||||
[Source](https://github.com/yaml/go-yaml/commit/1c17b9cee72bf81ae73a8be50fbd7dd3a9985125).
|
[Source](https://github.com/yaml/go-yaml/commit/f2f9563503f11b7a60da0c532175dcc693ce8d5a).
|
||||||
|
|
||||||
## golang.org/x/text
|
## golang.org/x/text
|
||||||
|
|
||||||
Selected module: `golang.org/x/text` at `v0.14.0`.
|
Selected module: `golang.org/x/text` at `v0.44.0`.
|
||||||
|
|
||||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
SPDX-License-Identifier: BSD-3-Clause
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-x-text.txt) |
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-x-text.txt) |
|
||||||
[LICENSE](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/LICENSE) |
|
[LICENSE](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/LICENSE) |
|
||||||
[PATENTS](https://github.com/golang/text/blob/6c97a165dd661335ff7bce6104a008558123c353/PATENTS).
|
[PATENTS](https://github.com/golang/text/blob/6070ac32abcdd1670599eeb3ee4d513c8184a513/PATENTS).
|
||||||
|
|
||||||
[Source](https://github.com/golang/text/commit/6c97a165dd661335ff7bce6104a008558123c353).
|
[Source](https://github.com/golang/text/commit/6070ac32abcdd1670599eeb3ee4d513c8184a513).
|
||||||
|
|
||||||
## jsonschema/v6
|
## jsonschema/v6
|
||||||
|
|
||||||
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.2`.
|
Selected module: `github.com/santhosh-tekuri/jsonschema/v6` at `v6.0.3`.
|
||||||
|
|
||||||
SPDX-License-Identifier: Apache-2.0
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
|
License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
|
||||||
|
|
||||||
[Source](https://github.com/santhosh-tekuri/jsonschema/commit/29cbed948d24a04700eb94436416b18a07953b71).
|
[Source](https://github.com/santhosh-tekuri/jsonschema/commit/b0fc661f4939578bc429f408c18202573dbafcc4).
|
||||||
|
|
||||||
## PromptKit
|
## PromptKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.11.2`.
|
Selected module: `go.maximumdirect.net/promptkit` at `v0.23.2`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
@@ -150,55 +221,34 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|||||||
|
|
||||||
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
|
||||||
[GPL v3](licenses/GPL-3.0.txt) |
|
[GPL v3](licenses/GPL-3.0.txt) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2/COPYRIGHT).
|
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/382ccfb14b9630fd8f48c8364dbe2b6159637d71/COPYRIGHT).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/12d6d80024e80c552d122a076c22429bf6f2f6c2).
|
[Source](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/382ccfb14b9630fd8f48c8364dbe2b6159637d71).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog
|
## Cobra
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` at `v2.20260926.0`.
|
Selected module: `github.com/spf13/cobra` at `v1.10.2`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright 2013-2023 The Cobra Authors
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
|
|
||||||
|
|
||||||
## Promptkit Rakestrawhome Catalog
|
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` at `v2.20260926.0`.
|
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
SPDX-License-Identifier: MIT
|
|
||||||
|
|
||||||
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
|
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
|
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
|
|
||||||
|
|
||||||
## yaml.v3
|
|
||||||
|
|
||||||
Selected module: `gopkg.in/yaml.v3` at `v3.0.1`.
|
|
||||||
|
|
||||||
Copyright (c) 2006-2010 Kirill Simonov
|
|
||||||
|
|
||||||
Copyright (c) 2006-2011 Kirill Simonov
|
|
||||||
|
|
||||||
Copyright (c) 2011-2019 Canonical Ltd
|
|
||||||
|
|
||||||
Copyright 2011-2016 Canonical Ltd.
|
|
||||||
|
|
||||||
SPDX-License-Identifier: Apache-2.0 AND MIT
|
|
||||||
|
|
||||||
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
|
||||||
[MIT](licenses/MIT-libyaml.txt) |
|
[LICENSE](https://github.com/spf13/cobra/blob/88b30ab89da2d0d0abb153818746c5a2d30eccec/LICENSE.txt).
|
||||||
[LICENSE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/LICENSE) |
|
|
||||||
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
|
|
||||||
|
|
||||||
[Source](https://github.com/go-yaml/yaml/commit/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
|
[Source](https://github.com/spf13/cobra/tree/88b30ab89da2d0d0abb153818746c5a2d30eccec).
|
||||||
|
|
||||||
|
## pflag
|
||||||
|
|
||||||
|
Selected module: `github.com/spf13/pflag` at `v1.1.0`.
|
||||||
|
|
||||||
|
Copyright (c) 2012 Alex Ogier. All rights reserved.
|
||||||
|
|
||||||
|
Copyright (c) 2012 The Go Authors. All rights reserved.
|
||||||
|
|
||||||
|
SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
|
||||||
|
[LICENSE](https://github.com/spf13/pflag/blob/f8a990678a6d44385313e9b19e40737f889f9b14/LICENSE).
|
||||||
|
|
||||||
|
[Source](https://github.com/spf13/pflag/tree/f8a990678a6d44385313e9b19e40737f889f9b14).
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
Copyright (c) 2009 The Go Authors. All rights reserved.
|
Copyright 2009 The Go Authors.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without
|
Redistribution and use in source and binary forms, with or without
|
||||||
modification, are permitted provided that the following conditions are
|
modification, are permitted provided that the following conditions are
|
||||||
@@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
|
|||||||
copyright notice, this list of conditions and the following disclaimer
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
in the documentation and/or other materials provided with the
|
in the documentation and/or other materials provided with the
|
||||||
distribution.
|
distribution.
|
||||||
* Neither the name of Google Inc. nor the names of its
|
* Neither the name of Google LLC nor the names of its
|
||||||
contributors may be used to endorse or promote products derived from
|
contributors may be used to endorse or promote products derived from
|
||||||
this software without specific prior written permission.
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2026 Eric Rakestraw
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
|
||||||
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
|
||||||
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
|
|
||||||
following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all copies or substantial
|
|
||||||
portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
|
|
||||||
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
|
|
||||||
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
||||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
||||||
USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
Copyright (c) 2006-2010 Kirill Simonov
|
|
||||||
Copyright (c) 2006-2011 Kirill Simonov
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
|
||||||
the Software without restriction, including without limitation the rights to
|
|
||||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
||||||
of the Software, and to permit persons to whom the Software is furnished to do
|
|
||||||
so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Base Policy Version:** 1.3
|
**Base Policy Version:** 1.3
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-15
|
||||||
**Upstream Source:** [Standard Repository Policies architecture](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
**Upstream Source:** [Standard Repository Policies architecture](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
|
||||||
|
|
||||||
## Purpose And Ownership
|
## Purpose And Ownership
|
||||||
@@ -72,13 +72,19 @@ hidden global state.
|
|||||||
|
|
||||||
## Correction and Concurrency Invariants
|
## Correction and Concurrency Invariants
|
||||||
|
|
||||||
|
- Module focus guides proposal generation; it is not an exclusive acceptance
|
||||||
|
boundary. Validators permit otherwise acceptable minor editorial cleanup even
|
||||||
|
when the originating module did not request it. Existing applicability,
|
||||||
|
protection, and meaning checks still apply. See
|
||||||
|
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md) for the empirical
|
||||||
|
rationale and rejected strict-scope alternative.
|
||||||
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
|
||||||
indexing, validator ordering, transcript mutation, reports, and serialization.
|
indexing, validator ordering, transcript mutation, reports, and serialization.
|
||||||
- Validate corrections before final mutation, including local proposal checks,
|
- Validate corrections before final mutation, including local proposal checks,
|
||||||
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
decision cardinality, and apply-time safety. Provider-side schema enforcement
|
||||||
does not replace these checks.
|
does not replace these checks.
|
||||||
- Keep LLM concurrency bounded and context-aware through PromptKit's engine-local
|
- Use PromptKit's context-aware, engine-local per-backend admission for
|
||||||
per-backend admission. Proposal and validation calls selecting the same backend
|
LLM concurrency. Proposal and validation calls selecting the same backend
|
||||||
share that capacity, including retries and repairs. Preserve deterministic
|
share that capacity, including retries and repairs. Preserve deterministic
|
||||||
collection order. Recoverable model-operation failures stay local to their
|
collection order. Recoverable model-operation failures stay local to their
|
||||||
proposal response or validation batch; fatal invariant/application failure
|
proposal response or validation batch; fatal invariant/application failure
|
||||||
@@ -113,20 +119,21 @@ subprocess callers. Preserve the stream and exit behavior defined by the
|
|||||||
[CLI contract](../cli.md) and the artifact handling defined by
|
[CLI contract](../cli.md) and the artifact handling defined by
|
||||||
[operations](../operations.md).
|
[operations](../operations.md).
|
||||||
|
|
||||||
Diagnostics, reports, and surfaced errors must not leak configured LLM secrets.
|
|
||||||
Configuration redaction and LLM payload/error redaction have separate
|
|
||||||
responsibilities and should remain separate. Implementation details belong in
|
|
||||||
[diagnostics and reporting](../internal/diagnostics-reporting.md).
|
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
|
Keep Audita dependency-light and prefer the Go standard library where no adopted
|
||||||
filesystems, synchronization, and tests. ConfigKit owns configuration loading,
|
library owns the boundary. ConfigKit owns configuration composition, presence,
|
||||||
presence, provenance, and secret resolution; PromptKit owns prompt/profile
|
recorded resolution, and provenance. PromptKit owns prompt, profile, backend,
|
||||||
loading, OpenAI-compatible transport, retries, repairs, and backend admission.
|
credential, transport, retry, repair, and admission behavior. CommandKit and its
|
||||||
Audita owns domain meaning, role selection, and deterministic processing.
|
integrations own command routing and their standard inspection and request
|
||||||
|
surfaces. Audita owns domain meaning, role selection, transcript processing,
|
||||||
|
and presentation of its processing results.
|
||||||
|
|
||||||
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
|
Pass resolved typed configuration to preflight and processing. Preflight checks
|
||||||
|
execution readiness before output preparation or model work; the output session
|
||||||
|
owns resources, cleanup, and publication. The glossary and readiness decision is
|
||||||
|
recorded in [ADR 0002](../adr/0002-glossary-configuration-and-preflight.md).
|
||||||
|
Do not introduce broad dependency-injection, workflow, logging, or plugin
|
||||||
frameworks without a concrete implemented need and focused tests.
|
frameworks without a concrete implemented need and focused tests.
|
||||||
|
|
||||||
## Testing and Documentation
|
## Testing and Documentation
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Documentation Policy
|
# Documentation Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.11
|
**Base Policy Version:** 1.12
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md)
|
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/documentation.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -85,8 +85,8 @@ secret values.
|
|||||||
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
|
||||||
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
| Architectural decision records | `docs/adr/` | Records of significant architectural decisions and supporting rationale. | Ordinary design choices, current behavior reference, implementation status, task sequencing. |
|
||||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, or future work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
|
||||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
|
||||||
|
|
||||||
@@ -154,8 +154,15 @@ Use sequentially numbered ADR filenames such as
|
|||||||
|
|
||||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
|
||||||
belong in the roadmap.
|
ADRs should be reserved for major architectural decisions, such as ownership
|
||||||
|
boundaries, large-scale architectural features, or invariants that rise to
|
||||||
|
the level of architectural significance. Routine choices, implementation
|
||||||
|
decisions, and rejected suggestions do not warrant an ADR unless they
|
||||||
|
otherwise represent a significant architectural decision.
|
||||||
|
|
||||||
|
Roadmaps should preserve plausible future work, not rejected product
|
||||||
|
ideas.
|
||||||
|
|
||||||
## Maintenance
|
## Maintenance
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Open Source Licensing And Attribution Policy
|
# Open Source Licensing And Attribution Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.8
|
**Base Policy Version:** 1.9
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
|
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/oss.md)
|
||||||
|
|
||||||
## Purpose And Scope
|
## Purpose And Scope
|
||||||
|
|
||||||
@@ -64,9 +64,12 @@ license unless the author deliberately changes it. For mixed-purpose projects,
|
|||||||
follow the author's selected scope rather than inferring different licenses for
|
follow the author's selected scope rather than inferring different licenses for
|
||||||
individual packages.
|
individual packages.
|
||||||
|
|
||||||
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/application/COPYRIGHT)
|
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
|
||||||
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/library/COPYRIGHT)
|
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
|
||||||
to create the project’s root `COPYRIGHT` file.
|
to create the project’s root `COPYRIGHT` file.
|
||||||
|
Use the COPYRIGHT examples exactly as provided, except for placeholder
|
||||||
|
substitutions and changes expressly required or authorized by this policy or
|
||||||
|
the author.
|
||||||
Use verified project names, contribution years, and copyright holders.
|
Use verified project names, contribution years, and copyright holders.
|
||||||
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
For another selected license, adapt the grant, SPDX expression, disclaimer, and
|
||||||
text references together to reflect the author's selected grant.
|
text references together to reflect the author's selected grant.
|
||||||
@@ -113,7 +116,7 @@ Each entry presents these five fields in order, with blank lines between them:
|
|||||||
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
|
||||||
5. **Source repository link**, rendered exactly as `[Source](COMMIT_URL).`.
|
5. **Source repository link**, rendered exactly as `[Source](COMMIT_URL).`.
|
||||||
|
|
||||||
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/attribution.md)
|
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/attribution.md)
|
||||||
to create the project’s `docs/oss/attribution.md`.
|
to create the project’s `docs/oss/attribution.md`.
|
||||||
The example illustrates omitted fields and multiple copyrights and licenses.
|
The example illustrates omitted fields and multiple copyrights and licenses.
|
||||||
Replace the uppercase placeholders and illustrative URLs with actual component
|
Replace the uppercase placeholders and illustrative URLs with actual component
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
# Release Policy
|
# Release Policy
|
||||||
|
|
||||||
**Base Policy Version:** 1.4
|
**Base Policy Version:** 1.7
|
||||||
|
|
||||||
**Last Updated**: 2026-09-28
|
**Last Updated**: 2026-11-11
|
||||||
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
|
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/release.md)
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
@@ -11,6 +11,9 @@ This policy defines a proportionate workflow for validating and publishing an
|
|||||||
intended source revision under a fresh version. Releases should require little
|
intended source revision under a fresh version. Releases should require little
|
||||||
routine intervention, avoid repeated work, and preserve published contents.
|
routine intervention, avoid repeated work, and preserve published contents.
|
||||||
|
|
||||||
|
Adopt stronger verification or reproducibility requirements through an explicit
|
||||||
|
project-specific decision justified by the intended audience and support needs.
|
||||||
|
|
||||||
The repository-specific commands belong in `docs/release.md`. Historical release
|
The repository-specific commands belong in `docs/release.md`. Historical release
|
||||||
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
summaries belong in `docs/releases/`. The procedure must agree with the adopted
|
||||||
policy and link to it rather than introduce additional release requirements.
|
policy and link to it rather than introduce additional release requirements.
|
||||||
@@ -49,12 +52,17 @@ failures and optional investigations under artifact-delivery status.
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Linux amd64 | Required production target |
|
| Linux amd64 | Required production target |
|
||||||
| Linux arm64 | Required production target |
|
| Linux arm64 | Required production target |
|
||||||
| macOS arm64 | Best effort for local development; failures do not block a release or Linux artifact publication |
|
| macOS arm64 | Best effort for local development; development failures do not block a release or Linux artifact publication |
|
||||||
|
|
||||||
Windows and macOS amd64 are unsupported. Neither their builds nor tests are
|
Windows and macOS amd64 are unsupported. Neither their builds nor tests are
|
||||||
release requirements. Native macOS testing and access to a Mac are not release
|
release requirements. Native macOS testing and access to a Mac are not release
|
||||||
prerequisites.
|
prerequisites.
|
||||||
|
|
||||||
|
Linux amd64 and arm64 remain the required production targets. Publication from
|
||||||
|
a host outside the selected target set must pass ReleaseKit's temporary native
|
||||||
|
version check. This check does not require access to a Mac or native tests on
|
||||||
|
every delivery target.
|
||||||
|
|
||||||
## Versioning And Compatibility
|
## Versioning And Compatibility
|
||||||
|
|
||||||
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
|
||||||
@@ -70,9 +78,18 @@ requires a deliberate update to the policy and procedure where needed.
|
|||||||
For releases at or after `v1.0.0`:
|
For releases at or after `v1.0.0`:
|
||||||
|
|
||||||
- a major release may make intentional incompatible changes;
|
- a major release may make intentional incompatible changes;
|
||||||
- a minor release may add backward-compatible behavior; and
|
- a minor release may add backward-compatible behavior or raise the Go consumer
|
||||||
|
minimum as described below; and
|
||||||
- a patch release may make backward-compatible fixes.
|
- a patch release may make backward-compatible fixes.
|
||||||
|
|
||||||
|
In this context, "backward-compatible" means only that pre-existing, valid
|
||||||
|
configurations will continue to work following the release. Behavior need not
|
||||||
|
remain strictly identical.
|
||||||
|
|
||||||
|
Audita's minor and patch releases also preserve the documented
|
||||||
|
[subprocess integration](../integrations/subprocess.md) and
|
||||||
|
[output-file](../integrations/output-files.md) contracts.
|
||||||
|
|
||||||
Before `v1.0.0`, a minor release may change a documented public, operational,
|
Before `v1.0.0`, a minor release may change a documented public, operational,
|
||||||
configuration, data, integration, or compatibility contract when the release
|
configuration, data, integration, or compatibility contract when the release
|
||||||
note explains the impact and required consumer action. A patch release must
|
note explains the impact and required consumer action. A patch release must
|
||||||
@@ -82,6 +99,36 @@ Additional compatibility promises and deliberate exceptions belong in the
|
|||||||
adopted policy or canonical contract documentation. Release notes explain
|
adopted policy or canonical contract documentation. Release notes explain
|
||||||
changes; they do not redefine current compatibility contracts.
|
changes; they do not redefine current compatibility contracts.
|
||||||
|
|
||||||
|
## Consumer Minimum And Release Toolchain
|
||||||
|
|
||||||
|
Distinguish the minimum toolchain supported for consumers from the toolchain
|
||||||
|
maintainers use to build and validate a release. Record the consumer minimum in
|
||||||
|
the language's standard metadata and explain its support meaning in canonical
|
||||||
|
documentation. Define the selected release toolchain and its setup commands in
|
||||||
|
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
|
||||||
|
choices.
|
||||||
|
|
||||||
|
For Go repositories, normally declare the initial stable release of the selected
|
||||||
|
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
|
||||||
|
when a dependency or an identified correctness fix requires it, and document
|
||||||
|
the reason.
|
||||||
|
|
||||||
|
For release-policy purposes, successful build and release validation on the
|
||||||
|
selected Go patch release are sufficient to presume support across that Go
|
||||||
|
series, subject to the declared consumer minimum and concrete evidence of
|
||||||
|
incompatibility. Investigate specific compatibility concerns as they arise.
|
||||||
|
|
||||||
|
Use the selected release toolchain for candidate validation and release builds,
|
||||||
|
including CI jobs performing those tasks. A newer validation patch version alone
|
||||||
|
leaves the consumer minimum unchanged.
|
||||||
|
|
||||||
|
A minor release may raise the Go consumer minimum, including from one Go series
|
||||||
|
to the next, both before and after `v1.0.0`. This is an explicit exception to the
|
||||||
|
backward-compatibility requirement for minor releases. Patch releases preserve
|
||||||
|
the consumer minimum. Explain an increase's impact and required consumer action
|
||||||
|
in the release note. Other incompatible changes remain subject to the versioning
|
||||||
|
rules above.
|
||||||
|
|
||||||
## Release Documentation And Preparation
|
## Release Documentation And Preparation
|
||||||
|
|
||||||
Every new release must include `docs/releases/<version>.md` in the tagged
|
Every new release must include `docs/releases/<version>.md` in the tagged
|
||||||
@@ -140,9 +187,19 @@ caches are not required. If the candidate changes, validate the new candidate.
|
|||||||
Cheap publication-state checks may be repeated immediately before pushing
|
Cheap publication-state checks may be repeated immediately before pushing
|
||||||
because local or remote state may have changed during validation.
|
because local or remote state may have changed during validation.
|
||||||
|
|
||||||
Automate mechanical guards. Do not require manual approvals, sign-offs, or
|
Automate mechanical guards. Human review gates are strongly disfavored: add one
|
||||||
routine confirmations as release gates. Version selection and
|
only through an explicit project-specific policy decision explaining the
|
||||||
a concise release summary remain preparation inputs, not separate approval rounds.
|
concrete risk and why ordinary change review or automation is insufficient.
|
||||||
|
Do not add routine confirmations for checks the command can perform itself.
|
||||||
|
|
||||||
|
## Audita's Selected Candidate Checks
|
||||||
|
|
||||||
|
Audita uses the module-selected ReleaseKit with race-enabled offline tests, full Go vet,
|
||||||
|
formatting, Markdown links, module distribution checks, and the required target
|
||||||
|
builds. Race checks address shared processing and client state; vet and format
|
||||||
|
checks catch common defects and mechanical drift; link and module checks protect
|
||||||
|
the distributed source tree. The [runbook](../release.md) specifies the selected
|
||||||
|
toolchain, native commands, and asynchronous recovery.
|
||||||
|
|
||||||
## Publication Guards
|
## Publication Guards
|
||||||
|
|
||||||
@@ -196,7 +253,8 @@ release without replacing its contents or reusing its version.
|
|||||||
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
|
||||||
identify:
|
identify:
|
||||||
|
|
||||||
1. prerequisites, primary branch, and authoritative remote;
|
1. prerequisites, selected release toolchain, primary branch, and authoritative
|
||||||
|
remote;
|
||||||
2. version and release-note preparation;
|
2. version and release-note preparation;
|
||||||
3. one publication command with its selected local checks and automated guards;
|
3. one publication command with its selected local checks and automated guards;
|
||||||
4. declared deliverables, required build targets, and automation responsibilities;
|
4. declared deliverables, required build targets, and automation responsibilities;
|
||||||
|
|||||||
242
docs/release.md
242
docs/release.md
@@ -1,144 +1,178 @@
|
|||||||
# Releasing Audita
|
# Releasing Audita
|
||||||
|
|
||||||
The [release policy](policy/release.md) defines versioning, supported platforms,
|
The [release policy](policy/release.md) owns release requirements. This runbook
|
||||||
immutability, and success. This runbook implements it with
|
uses the native commands in [scripts/release.sh](../scripts/release.sh), which
|
||||||
[scripts/release.sh](../scripts/release.sh). Publication ends after a successful
|
invokes the module-selected ReleaseKit tool. A confirmed source tag completes
|
||||||
source/tag push; binary delivery runs asynchronously.
|
the release; Woodpecker reports binary delivery separately.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
Use Bash, Git, and Go 1.25.5 or newer (CI uses Go 1.25.5), with dependencies
|
Use Bash, Git, a C compiler, and cgo for race detection. The authoritative
|
||||||
available locally or downloadable. Tests themselves are offline. Work from a
|
remote is `origin`, which must point to `ssh://git@forge.maximumdirect.net:22222/eric/audita.git`; the
|
||||||
checkout of `main` with permission to push that branch and tags to `origin`.
|
primary branch is `main`. Source publication needs no delivery credential.
|
||||||
For this repository, `origin` is `https://gitea.maximumdirect.net/eric/audita.git`.
|
Woodpecker uses the existing `GITEA_RELEASE_TOKEN` secret for delivery.
|
||||||
The script uses the checkout's configured `origin`; set it correctly before use.
|
|
||||||
The remote must support atomic pushes. No CI credentials or Mac are needed locally.
|
|
||||||
|
|
||||||
CI uses the existing `GITEA_RELEASE_TOKEN` secret for asynchronous asset delivery.
|
The source-build minimum is Go 1.27.0, declared in [go.mod](../go.mod).
|
||||||
macOS arm64 is a best-effort development environment; no macOS binary is published.
|
This minimum is required by the selected application dependencies.
|
||||||
|
Distributed executables do not require a Go installation. The selected release
|
||||||
|
toolchain is Go 1.27.4, matching the current Woodpecker image and distributed
|
||||||
|
runtime notices. Use that local installation for tool bootstrap, candidate
|
||||||
|
validation, and release builds. Replace the installation path below with its
|
||||||
|
actual directory:
|
||||||
|
|
||||||
## Prepare the Candidate
|
```sh
|
||||||
|
export PATH="/path/to/go1.27.4/bin:$PATH"
|
||||||
|
export GOTOOLCHAIN=local
|
||||||
|
go version
|
||||||
|
```
|
||||||
|
|
||||||
Choose an unused version following the policy. Write
|
Record the reported toolchain with the validation results. The
|
||||||
`docs/releases/<version>.md`, beginning with `# Audita <version>`, followed by a
|
[release policy](policy/release.md#consumer-minimum-and-release-toolchain)
|
||||||
short summary and any compatibility or upgrade instructions relevant to the
|
governs the distinction between the consumer minimum and release toolchain,
|
||||||
release. There are no mandatory boilerplate sections. Update affected canonical
|
support across a Go series, and versioning of minimum increases. When selecting
|
||||||
documentation with behavior changes.
|
a different release toolchain, update the CI image and applicable runtime
|
||||||
|
attribution together; a newer build patch alone need not raise the minimum.
|
||||||
|
|
||||||
End each new note with these links, substituting the selected release version for
|
Prepare dependencies before offline candidate validation:
|
||||||
`<version>`:
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off GOTOOLCHAIN=local go mod download
|
||||||
|
```
|
||||||
|
|
||||||
|
If dependency preparation changes `go.mod` or `go.sum`, include those changes in
|
||||||
|
the committed candidate. ReleaseKit's CLI is registered as a Go tool dependency;
|
||||||
|
`go mod tidy` retains its dependencies and checksums. The release wrapper changes
|
||||||
|
to the repository root, selects the host target, and invokes the full
|
||||||
|
`go.maximumdirect.net/releasekit/cmd/releasekit` tool path with workspace
|
||||||
|
discovery and automatic toolchain downloads disabled. Go builds or reuses the selected
|
||||||
|
tool from its normal build cache. Tool preparation precedes offline candidate
|
||||||
|
validation.
|
||||||
|
|
||||||
|
## Prepare and publish the candidate
|
||||||
|
|
||||||
|
Choose an unused version under the [release policy](policy/release.md), write
|
||||||
|
`docs/releases/<version>.md`, and begin it with `# Audita <version>`. Include a
|
||||||
|
short summary, relevant compatibility or upgrade instructions, and the required
|
||||||
|
`COPYRIGHT | LICENSE | ATTRIBUTION` footer with links for the selected version.
|
||||||
|
Update affected canonical documentation with behavior changes.
|
||||||
|
|
||||||
|
For a new major release at v2 or later, migrate the Go module path before
|
||||||
|
committing the candidate:
|
||||||
|
|
||||||
|
1. Change the module path in `go.mod` to end in the selected major suffix, such
|
||||||
|
as `/v4` for v4.0.0.
|
||||||
|
2. Update imports of Audita packages throughout application and test code to
|
||||||
|
match that module path. Update module-path references in build configuration
|
||||||
|
and scripts where applicable.
|
||||||
|
3. Update current installation examples and release-note upgrade instructions
|
||||||
|
to use the new module path and version. Preserve historical release notes.
|
||||||
|
4. Run `go mod tidy` and include the resulting module metadata in the candidate.
|
||||||
|
|
||||||
|
For example, the v4.0.0 cutover changes
|
||||||
|
`gitea.maximumdirect.net/eric/audita/v3` to
|
||||||
|
`gitea.maximumdirect.net/eric/audita/v4`. The publication command validates the
|
||||||
|
prepared candidate; module migration is a source change made during preparation.
|
||||||
|
The normal candidate tests and target builds below validate the migrated source.
|
||||||
|
|
||||||
|
End each note with this footer, substituting its complete tag for `<version>`:
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/docs/oss/attribution.md)
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/docs/oss/attribution.md)
|
||||||
```
|
```
|
||||||
|
|
||||||
Check that all three paths exist in the candidate source. Retire completed
|
Commit the complete candidate on `main`, including the release note, then push
|
||||||
roadmaps when useful during preparation, preserving planned and deferred work
|
the branch first:
|
||||||
and repairing affected links. This is ordinary preparation, not a script gate.
|
|
||||||
|
|
||||||
Commit the intended changes, including the note, on `main`. The release script
|
|
||||||
requires a clean index and worktree and does not create commits. There is no
|
|
||||||
separate approval round. Published notes describe their tagged release; existing
|
|
||||||
release history does not need to be reconstructed.
|
|
||||||
|
|
||||||
## Validate and Publish
|
|
||||||
|
|
||||||
From the repository root, substitute the chosen version:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
bash scripts/release.sh --publish v0.2.0
|
git push origin main
|
||||||
```
|
```
|
||||||
|
|
||||||
This single invocation:
|
Record the candidate commit. Before publication, remote `main` must identify
|
||||||
|
that exact commit. ReleaseKit checks this along with the clean candidate, note,
|
||||||
|
version, and remote tag state.
|
||||||
|
|
||||||
1. Checks the version, branch, clean candidate, matching note title, and tag absence.
|
Publish from the repository root, substituting the selected version:
|
||||||
2. Runs the default offline test suite once, then builds both required Linux targets
|
|
||||||
in a temporary directory using the same build script as CI.
|
|
||||||
3. Checks that the candidate and worktree are unchanged and that it includes the
|
|
||||||
current `origin/main` history.
|
|
||||||
4. Creates the version tag at that commit and atomically pushes the exact source
|
|
||||||
commit to `main` and the exact tag to `origin`.
|
|
||||||
5. Reports release success and exits without polling CI or checking asset availability.
|
|
||||||
|
|
||||||
To validate a prepared candidate without remote access, creating a tag, or pushing:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
bash scripts/release.sh --check v0.2.0
|
bash scripts/release.sh publish VERSION
|
||||||
```
|
```
|
||||||
|
|
||||||
This is an optional diagnostic command, not a required step before `--publish`.
|
The command runs the offline Go test suite with race detection, full Go vet,
|
||||||
Publication validates within its own invocation; it does not use persistent cached
|
formatting, Markdown link, and module distribution checks. Race detection covers
|
||||||
results. Both modes require the candidate note and a clean `main` checkout.
|
shared processing and client state; vet and formatting catch common Go defects and
|
||||||
|
mechanical drift; the link and module checks protect the distributed source tree. It builds the
|
||||||
|
profile's Linux amd64 and arm64 targets and checks the native version output. It creates an annotated tag whose
|
||||||
|
message comes from the committed note, then pushes the exact tag object. A
|
||||||
|
successful publication completes the source release without waiting for CI or
|
||||||
|
checking artifact availability.
|
||||||
|
|
||||||
## Artifact Delivery
|
`bash scripts/release.sh check VERSION` is an optional local diagnostic. It
|
||||||
|
does not inspect remote availability or publish a tag, and it is not a required
|
||||||
|
step before `publish`, which performs its own candidate validation.
|
||||||
|
|
||||||
[Woodpecker](../.woodpecker/release.yml) runs on tag events. It invokes
|
The wrapper preserves ReleaseKit's exit status: `0` means confirmed publication;
|
||||||
[scripts/build-release.sh](../scripts/build-release.sh), then publishes:
|
`1` means a known validation or publication failure/conflict; `2` means usage,
|
||||||
|
profile, or version input error; and `3` means the push outcome remains unknown.
|
||||||
|
|
||||||
|
## Binary delivery
|
||||||
|
|
||||||
|
Woodpecker runs ReleaseKit's `ci-deliver` command for tags under
|
||||||
|
`refs/tags/v*`. The tagged `go.mod` selects the ReleaseKit version, and
|
||||||
|
`releasekit.json` supplies Audita's build and delivery settings:
|
||||||
|
|
||||||
- `audita-<version>-linux-amd64`
|
- `audita-<version>-linux-amd64`
|
||||||
- `audita-<version>-linux-arm64`
|
- `audita-<version>-linux-arm64`
|
||||||
- `SHA256SUMS`
|
- `SHA256SUMS`
|
||||||
|
|
||||||
Builds disable CGO and use the shared script's release flags. The version is in the
|
Before running Go, the workflow fetches and verifies the event tag, checks it
|
||||||
asset name and source tag; Audita has no CLI version command. Cross-compilation
|
out detached, and confirms the source tree is clean. It peels the event object
|
||||||
checks compilation, not native execution on each architecture.
|
ID to its commit before comparing it with the tag's commit, accepting both
|
||||||
|
annotated tag objects and commit IDs from Woodpecker. It copies the tagged
|
||||||
|
`go.mod` and `go.sum` into temporary module metadata outside the checkout. Go
|
||||||
|
confirms that this configuration selects ReleaseKit, then invokes its CLI
|
||||||
|
using an alternate `-modfile` and the host target. Temporary metadata remains
|
||||||
|
outside the tagged source. The workflow runs with the selected Go 1.27.4 image,
|
||||||
|
`GOWORK=off`, and `GOTOOLCHAIN=local`.
|
||||||
|
|
||||||
The CI job status and logs show delivery progress or failure. Assets appear on
|
Delivery uses prepared-checkout mode to verify the existing tag, HEAD, and clean
|
||||||
the matching Gitea release page when delivery succeeds. A successful tag push
|
worktree without repeating the bootstrap's fetch and checkout.
|
||||||
must not be reported as confirmation that these files are available.
|
|
||||||
For a new hosted release, the release plugin uses the checked-in note at the
|
|
||||||
tag as its body, including the footer. On delivery retries it preserves
|
|
||||||
existing release information and skips files that already exist.
|
|
||||||
|
|
||||||
Consumers can download the binary for their Linux architecture and `SHA256SUMS`
|
The resulting executable receives:
|
||||||
from that page, check its listed SHA-256 digest with `sha256sum`, and make it
|
|
||||||
executable. Optional consumer checks are not maintainer release gates. Digests
|
|
||||||
detect file mismatches; they are not an independent signature of authenticity.
|
|
||||||
|
|
||||||
## Failure and Retry
|
|
||||||
|
|
||||||
**Local validation fails:** correct the issue, commit any changes, and rerun the
|
|
||||||
publication command. A failure before tag creation publishes nothing. Temporary
|
|
||||||
local build files are removed on exit.
|
|
||||||
|
|
||||||
**Push fails or the connection is interrupted:** the script retains the local tag
|
|
||||||
and reports failure rather than guessing whether publication succeeded. Inspect
|
|
||||||
remote state with the actual version:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git rev-parse 'v0.2.0^{commit}'
|
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
|
||||||
git ls-remote origin refs/heads/main refs/tags/v0.2.0
|
--repo "$${CI_WORKSPACE}" \
|
||||||
|
--prepared \
|
||||||
|
--commit "$${expected_commit}" \
|
||||||
|
"$${CI_COMMIT_TAG}"
|
||||||
```
|
```
|
||||||
|
|
||||||
If the remote tag identifies the intended candidate and its source is on remote
|
The workflow maps the existing `GITEA_RELEASE_TOKEN` secret to
|
||||||
`main`, the release succeeded; no CI waiting is needed. An atomic push normally
|
`RELEASEKIT_GITEA_TOKEN`. The complete checked-in release note, including its
|
||||||
updates both refs or neither. If `main` has advanced since publication, fetch it
|
licensing footer, supplies the hosted release body. Delivery runs independently
|
||||||
and check candidate ancestry rather than requiring tip equality.
|
after source publication; its status and logs report delivery progress or
|
||||||
|
failure.
|
||||||
|
|
||||||
If the tag is absent remotely and was never published or used by a release
|
ReleaseKit handles content-aware retries and retained-artifact recovery. See
|
||||||
channel, remove only that unpublished local tag (`git tag -d v0.2.0`), resolve
|
the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/publish.md)
|
||||||
any rejected branch update, and rerun the command. If the remote tag has a
|
and [delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/delivery.md)
|
||||||
different target, do not overwrite it; choose a fresh version. Never force a
|
for those mechanics. A delivery failure does not undo a confirmed source
|
||||||
branch or tag update to bypass the guards.
|
release. Retry the same immutable tag after resolving the reported state; do not
|
||||||
|
replace published files or a checksum manifest.
|
||||||
|
|
||||||
**CI delivery fails:** the release remains successful. Inspect the failed tag
|
## Failure and recovery
|
||||||
job's logs and retry that same tagged job for a transient failure. Publication is
|
|
||||||
configured to skip existing filenames and not overwrite assets. Keep the same
|
|
||||||
source and build environment when retrying; inspect any existing assets/checksum
|
|
||||||
manifest before filling gaps. Do not regenerate and replace a published manifest
|
|
||||||
or binary. If an existing manifest cannot describe the missing rebuilt assets,
|
|
||||||
use a new version instead. A pipeline or source defect also requires a corrected
|
|
||||||
commit and new version. An outage does not require moving the tag.
|
|
||||||
|
|
||||||
**A released source defect is discovered:** fix it on `main` and publish a new
|
Fix validation failures in a new committed candidate, then rerun `publish`.
|
||||||
appropriate version with a note explaining the correction. Leave the old tag and
|
After a failed push, ReleaseKit retains the local annotated tag and reports the
|
||||||
published files intact.
|
captured authoritative endpoint and publication state. Resolve the result
|
||||||
|
against that endpoint before retrying. An exact remote tag-object match, with
|
||||||
|
its source available on remote `main`, can confirm publication. If the remote
|
||||||
|
state cannot be read, the result remains unknown; do not remove or replace the
|
||||||
|
tag while uncertain. Follow the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/publish.md)
|
||||||
|
for object-level recovery details.
|
||||||
|
|
||||||
## Maintaining the Release Workflow
|
A source-release success and artifact-delivery status are separate. For a
|
||||||
|
partial or uncertain delivery, inspect the job output and follow the pinned
|
||||||
The shared build script keeps local and CI build settings aligned. Tests in
|
[delivery recovery guidance](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.10.0/docs/consumers/delivery.md).
|
||||||
[scripts/release_test.go](../scripts/release_test.go) exercise publication against
|
A source or published-content correction requires a new appropriate version;
|
||||||
temporary local Git remotes, including guard failures and rejected pushes, with
|
leave the original tag and its contents intact.
|
||||||
Go execution stubbed to avoid recursive builds/tests. They run in `go test ./...`.
|
|
||||||
Changes to CI delivery should preserve explicit asset names, checksums, and
|
|
||||||
non-overwriting publication. CI delivery is separate from the local success path.
|
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Release Notes
|
# Release Notes
|
||||||
|
|
||||||
|
- [Audita v4.0.0](v4.0.0.md)
|
||||||
|
- [Audita v3.1.1](v3.1.1.md)
|
||||||
|
- [Audita v3.1.0](v3.1.0.md)
|
||||||
|
- [Audita v3.0.1](v3.0.1.md)
|
||||||
|
- [Audita v3.0.0](v3.0.0.md)
|
||||||
- [Audita v2.0.1](v2.0.1.md)
|
- [Audita v2.0.1](v2.0.1.md)
|
||||||
- [Audita v2.0.0](v2.0.0.md)
|
- [Audita v2.0.0](v2.0.0.md)
|
||||||
- [Audita v1.1.0](v1.1.0.md)
|
- [Audita v1.1.0](v1.1.0.md)
|
||||||
|
|||||||
59
docs/releases/v3.0.0.md
Normal file
59
docs/releases/v3.0.0.md
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Audita v3.0.0
|
||||||
|
|
||||||
|
Audita v3 adopts native configuration inheritance and layered PromptKit
|
||||||
|
definitions, and allows transcript and report destinations to be selected
|
||||||
|
independently. This is an intentional breaking release: existing PromptKit
|
||||||
|
profile files require migration from the v2 release format.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- ConfigKit is updated from v0.4.2 to v0.5.0. Application configuration supports
|
||||||
|
native sibling inheritance, and configuration explanations include value-free
|
||||||
|
inheritance provenance.
|
||||||
|
- PromptKit is updated from v0.11.2 to v0.12.0, with both backend catalogs at
|
||||||
|
`v3.20260930.0`. Native keyed definitions support profile inheritance and
|
||||||
|
operator prompt overrides. The selected `profiles_dir` can contain profiles,
|
||||||
|
backends, and prompts; application backend endpoints, capacity, and credentials
|
||||||
|
remain authoritative.
|
||||||
|
- Effective definitions and referenced prompt message contents are captured at
|
||||||
|
runtime construction. Consumed host configuration and message files are
|
||||||
|
protected from output collisions. Audita retains its per-call schemas, domain
|
||||||
|
validators, retry and repair budgets, and resilient processing behavior.
|
||||||
|
- An explicit `--report-dir` now selects reports for either file or stdout
|
||||||
|
transcript output. File output without that flag still places reports beside
|
||||||
|
the transcript. This supports Narratio's independent output destinations;
|
||||||
|
debug capture remains independently selectable.
|
||||||
|
|
||||||
|
## Upgrading from v2
|
||||||
|
|
||||||
|
1. Convert each old PromptKit profile to a keyed definition under
|
||||||
|
`profiles: {<profile-id>: ...}`, remove its inner `id`, and replace
|
||||||
|
`base_profile` with `extends`. Profile IDs come from mapping keys rather than
|
||||||
|
filenames. No compatibility loader or automatic conversion is provided.
|
||||||
|
2. Review explicit zero, omitted, and null generation controls. For example,
|
||||||
|
`temperature: 0` now sends an explicit zero, omission leaves the control
|
||||||
|
unspecified, and null suppresses an inherited control. A zero profile timeout
|
||||||
|
disables PromptKit's generation deadline; Audita's operation deadline still
|
||||||
|
applies.
|
||||||
|
3. Select a configuration root whose final definitions are valid. PromptKit now
|
||||||
|
validates unused definitions too, including backend references, and conflicting
|
||||||
|
values within a directory source group are errors. The maintained examples
|
||||||
|
use separate [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/local/) and
|
||||||
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/service/) roots. Runtime source edits take
|
||||||
|
effect on the next invocation.
|
||||||
|
4. Rename any application backend literally named `extends` and update its
|
||||||
|
profile references. ConfigKit now reserves that key for inheritance.
|
||||||
|
5. Validate the migrated installation with
|
||||||
|
`audita config validate --config /path/to/config.yml`, including any selected
|
||||||
|
testing overlay. Application configuration continues to use `version: 2`.
|
||||||
|
|
||||||
|
The [v3.0.0 configuration reference](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/config.md#promptkit-profiles-and-operator-prompts)
|
||||||
|
contains the migrated format and presence rules. Start from the
|
||||||
|
[local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/minimal-config.yml),
|
||||||
|
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/production-config.yml), or
|
||||||
|
[application inheritance](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/configkit-inheritance.yml) examples.
|
||||||
|
The [CLI reference](../cli.md) describes report selection, and the
|
||||||
|
[release procedure](../release.md) describes source publication and asynchronous
|
||||||
|
Linux binary delivery.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/oss/attribution.md)
|
||||||
14
docs/releases/v3.0.1.md
Normal file
14
docs/releases/v3.0.1.md
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# Audita v3.0.1
|
||||||
|
|
||||||
|
This patch release adds a configuration-free version command and adopts
|
||||||
|
ReleaseKit for source publication and Linux binary delivery. Transcript
|
||||||
|
processing, configuration, and LLM behavior are unchanged.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- `audita version` prints the application version without loading application
|
||||||
|
configuration or contacting an LLM provider.
|
||||||
|
- Release publication and Linux artifact delivery now use ReleaseKit v0.1.0.
|
||||||
|
The release runbook describes the maintainer workflow and delivery recovery.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.1/docs/oss/attribution.md)
|
||||||
42
docs/releases/v3.1.0.md
Normal file
42
docs/releases/v3.1.0.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# Audita v3.1.0
|
||||||
|
|
||||||
|
This minor release updates Audita's configuration, LLM, and release libraries
|
||||||
|
and raises the minimum Go version for source builds to 1.26.0. Existing Audita
|
||||||
|
configuration files, PromptKit profiles, and command behavior require no
|
||||||
|
migration.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- ConfigKit is updated from v0.5.0 to v0.7.1, with improvements to configuration
|
||||||
|
loading, resolution, provenance, and inspection efficiency.
|
||||||
|
- PromptKit is updated from v0.12.0 to v0.14.0, reducing avoidable allocations in
|
||||||
|
preparation, rendering, and execution. Audita retains its existing schemas,
|
||||||
|
validation, retry and repair budgets, and resilient processing behavior.
|
||||||
|
The libraries' new discovery and configuration-comparison APIs remain unused
|
||||||
|
by Audita.
|
||||||
|
- ReleaseKit is updated from v0.1.0 to v0.4.1. The local release launcher now
|
||||||
|
invokes the module-selected Go tool, and CI selects the tool from the tagged
|
||||||
|
module metadata. Prepared-checkout delivery reuses the tag checkout already
|
||||||
|
verified during CI bootstrap. Source publication still completes at a
|
||||||
|
successful tag push; Linux binary delivery remains asynchronous.
|
||||||
|
- Release builds and CI use Go 1.26.8, with corresponding runtime attribution
|
||||||
|
updated. The source-build minimum is tracked separately from that build
|
||||||
|
toolchain patch version.
|
||||||
|
|
||||||
|
## Upgrading from v3.0.1
|
||||||
|
|
||||||
|
Source builders must upgrade from the previous Go 1.25.5 minimum to **Go 1.26.0
|
||||||
|
or newer**, including any build environments still using Go 1.25. Maintainers
|
||||||
|
should follow the updated [release procedure](../release.md) for toolchain setup,
|
||||||
|
publication, and delivery recovery.
|
||||||
|
|
||||||
|
Go module consumers must use the v3 module path. For example:
|
||||||
|
`go install gitea.maximumdirect.net/eric/audita/v3/cmd/audita@v3.1.0`. This
|
||||||
|
aligns the module path with the v3 release line. Downloaded binaries and
|
||||||
|
commands run from a source checkout are unaffected.
|
||||||
|
|
||||||
|
Users of distributed Linux binaries need no Go installation. No configuration,
|
||||||
|
profile, transcript-format, or command-behavior migration is required for this
|
||||||
|
release.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/docs/oss/attribution.md)
|
||||||
20
docs/releases/v3.1.1.md
Normal file
20
docs/releases/v3.1.1.md
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# Audita v3.1.1
|
||||||
|
|
||||||
|
This patch fixes the Woodpecker bootstrap failure that prevented Linux binary
|
||||||
|
delivery for v3.1.0. Audita's transcript processing and configuration are unchanged.
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
|
||||||
|
- Resolve Woodpecker's tag-event object ID to its source commit before checking
|
||||||
|
the tagged checkout and invoking ReleaseKit. Annotated tag events previously
|
||||||
|
failed a comparison between the tag object and commit, stopping the workflow
|
||||||
|
before binary builds began.
|
||||||
|
- Add offline regression coverage for annotated and lightweight tag events,
|
||||||
|
including rejection of an unrelated source commit.
|
||||||
|
|
||||||
|
No configuration, profile, transcript-format, or command-behavior migration is
|
||||||
|
required from v3.1.0. The Go source-build minimum remains 1.26.0. Linux amd64 and
|
||||||
|
arm64 binary delivery follows the existing asynchronous
|
||||||
|
[release procedure](../release.md#binary-delivery).
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/docs/oss/attribution.md)
|
||||||
95
docs/releases/v4.0.0.md
Normal file
95
docs/releases/v4.0.0.md
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
# Audita v4.0.0
|
||||||
|
|
||||||
|
Audita v4.0.0 adopts application configuration schema v3 and native library
|
||||||
|
commands, adds repair of rejected corrections, and improves processing
|
||||||
|
reliability and resource use. This major release requires migration from
|
||||||
|
Audita v3.1.x configurations and command invocations.
|
||||||
|
|
||||||
|
## Configuration and command migration
|
||||||
|
|
||||||
|
- Replace `pipeline.modules` with ordered `pipeline.stages` records. Each entry
|
||||||
|
can select its own profiles, glossary context, protected terms, validation
|
||||||
|
budget, and transcript description. Optional stage IDs identify repeated
|
||||||
|
modules in reports and debug evidence.
|
||||||
|
- Move standalone glossaries into named `glossary_collections`, composed through
|
||||||
|
ConfigKit imports and companion files. Select glossary context and protected
|
||||||
|
terms independently. Replace `--glossary` and `--modules` invocations with the
|
||||||
|
new configuration and collection-selection flags.
|
||||||
|
- Move profiles and backends into native PromptKit resources. Application
|
||||||
|
configuration discovery, overlays, imports, and prompt companions now follow
|
||||||
|
the adopted library conventions. Processing preflight checks the entire
|
||||||
|
selected pipeline before preparing outputs or invoking a model.
|
||||||
|
- Replace `config print-effective` and the curated profile summary with native
|
||||||
|
configuration and PromptKit inspection commands. Help lists the available
|
||||||
|
inspection, comparison, preview, validation, and explicit provider smoke
|
||||||
|
commands.
|
||||||
|
- Remove confidence thresholds and confidence fields from custom prompts and
|
||||||
|
response examples. Audita now decides whether to apply corrections through
|
||||||
|
validation without model-supplied confidence scores.
|
||||||
|
- Update automation for CommandKit reporting modes. Completion counts require
|
||||||
|
`--verbose` or `--debug`. Quiet processing requires a transcript file
|
||||||
|
destination or `/dev/null`. `--debug` enables retained diagnostic artifacts;
|
||||||
|
`--progress` shows stage, chunk, proposal, validation, and repair activity on
|
||||||
|
stderr.
|
||||||
|
|
||||||
|
Follow the [configuration migration guide](../migrations/v3.md) and
|
||||||
|
[maintained examples](../../examples/), then inspect the resolved configuration
|
||||||
|
and try a small transcript. The migration guide's v3 refers to the application
|
||||||
|
configuration schema. The [CLI reference](../cli.md) owns current commands and
|
||||||
|
exit semantics.
|
||||||
|
|
||||||
|
## Correction quality and recovery
|
||||||
|
|
||||||
|
Rejected proposals can be revised using validator feedback and the original
|
||||||
|
proposal conversation. Correction repair defaults to one attempt per proposal,
|
||||||
|
can use a separate profile, and can be disabled. Every revision passes through
|
||||||
|
validation again. Withdrawn, exhausted, and unavailable branches finish
|
||||||
|
independently while other approved corrections can proceed.
|
||||||
|
|
||||||
|
Prompts clarify module focus, incidental editorial cleanup, protected names,
|
||||||
|
span deletion, and repair outcomes. Missing required validator decisions continue
|
||||||
|
to affect only their candidates. Recoverable model failures can coexist with a
|
||||||
|
successful run and applied corrections. See [configuration](../config.md#settings)
|
||||||
|
for budgets and profile selection.
|
||||||
|
|
||||||
|
## Reliability and reports
|
||||||
|
|
||||||
|
Processing uses a bounded window of sections within each stage, preserves
|
||||||
|
deterministic correction identities, and commits stage text after successful
|
||||||
|
execution. Fatal failures and cancellation stop and join active workers.
|
||||||
|
Transcript decoding, normalization, chunking, glossary preparation, and evidence
|
||||||
|
retention use less repeated work and fewer unnecessary copies. An explicit
|
||||||
|
target section count now produces exactly that many nonempty sections when
|
||||||
|
feasible and reports infeasible requests as input errors.
|
||||||
|
|
||||||
|
Output publication uses checked destinations and retains processing and delivery
|
||||||
|
failure evidence. Ordinary reports retain schema version 1 and add stage IDs,
|
||||||
|
correction repair outcomes, and available peer interruption details. See the
|
||||||
|
[output contract](../integrations/output-files.md) and
|
||||||
|
[operations guide](../operations.md) for report interpretation and output
|
||||||
|
lifecycle.
|
||||||
|
|
||||||
|
Audita no longer performs application-wide credential substitution in diagnostic
|
||||||
|
messages or artifacts. Review diagnostic handling before deployment and protect
|
||||||
|
retained debug data according to the sensitivity of its transcript, prompt, and
|
||||||
|
response content. Native library inspection retains its own rendering behavior.
|
||||||
|
Debug artifacts have changed, including removal of confidence fields and the
|
||||||
|
Audita-owned utilization diagnostics file.
|
||||||
|
|
||||||
|
## Dependencies and source builds
|
||||||
|
|
||||||
|
Source builds require Go 1.27.0 or newer. Release builds and Woodpecker use
|
||||||
|
Go 1.27.4. Distributed Linux binaries require no Go installation.
|
||||||
|
|
||||||
|
Go module installations use the new `/v4` module path. For example:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go install gitea.maximumdirect.net/eric/audita/v4/cmd/audita@v4.0.0
|
||||||
|
```
|
||||||
|
|
||||||
|
This release selects ConfigKit v1.2.1, PromptKit v0.23.2, CommandKit v0.7.0,
|
||||||
|
ckconfig v0.9.0, ckprompt v0.10.0, ReleaseKit v0.10.0, and stable go-yaml v4.0.0.
|
||||||
|
Audita's direct YAML imports now use v4. Linux amd64 and arm64 remain the
|
||||||
|
production targets; macOS arm64 receives best-effort development support.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v4.0.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v4.0.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v4.0.0/docs/oss/attribution.md)
|
||||||
@@ -1,171 +0,0 @@
|
|||||||
# ConfigKit Adoption Roadmap
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
**Status:** Production implementation complete. The selected configuration and
|
|
||||||
overlay model is implemented; [implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
|
||||||
records verification. No release has been published.
|
|
||||||
|
|
||||||
## Purpose and Direction
|
|
||||||
|
|
||||||
Adopt ConfigKit to harmonize Audita's configuration policy with the other
|
|
||||||
applications in the suite. Consistent configuration conventions are the primary
|
|
||||||
benefit; reducing application-owned configuration infrastructure is a secondary
|
|
||||||
benefit and a useful measure of integration quality.
|
|
||||||
|
|
||||||
This is an intentional breaking migration. There is no goal of backward
|
|
||||||
compatibility with existing installations, configuration files, CLI flags,
|
|
||||||
environment variables, or configuration-inspection output. Provide a migration
|
|
||||||
guide instead of legacy loaders, aliases, dual operating modes, or an ongoing
|
|
||||||
compatibility layer. Retain existing syntax only when it independently suits the
|
|
||||||
new model, not because compatibility requires it.
|
|
||||||
|
|
||||||
The initial review and prototype used ConfigKit v0.3.1, then verified v0.4.0
|
|
||||||
selected overlays. Those findings remain historical; the prototype was removed
|
|
||||||
after production integration. The application depends on a versioned module,
|
|
||||||
without a sibling checkout or local module replacement.
|
|
||||||
|
|
||||||
The shared boundary/prototype unit is complete. Its [selected contract, inventory,
|
|
||||||
and evidence](library-boundary-findings.md) define the detailed target alongside
|
|
||||||
this roadmap. Production configuration migration is complete.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
- ConfigKit owns strict YAML loading, input presence, default application, explicit
|
|
||||||
override mechanics, recorded resolution, secret references, and safe inspection.
|
|
||||||
- Audita owns its configuration types, field meanings, defaults, schema-version
|
|
||||||
policy, file selection, CLI parsing, domain validation, and runtime initialization.
|
|
||||||
- Adopt ConfigKit's precedence and validation lifecycle rather than reproduce
|
|
||||||
Audita's current per-layer validation and environment-overlay behavior.
|
|
||||||
- General configuration environment overrides are retired. Credentials use typed
|
|
||||||
environment-name secret references; supplied references must resolve according
|
|
||||||
to ConfigKit's contract. File selection follows the application-owned
|
|
||||||
discovery rules in the selected boundary.
|
|
||||||
- Standardize duration representations, explicit empty/null/zero handling, and
|
|
||||||
inheritance on the shared conventions. Express genuine Audita domain rules
|
|
||||||
through the supported resolution and validation APIs.
|
|
||||||
- Retained domain-setting flags and proposal/validation profile flags become
|
|
||||||
explicit ConfigKit overrides according to the migration inventory. Remove old
|
|
||||||
provider/key/retry/concurrency flags and aliases. Credentials use secret references.
|
|
||||||
- Use ConfigKit's redacted effective output and provide source/explanation access
|
|
||||||
through the selected views below. Legacy JSON inspection is replaced.
|
|
||||||
Diagnostic configuration snapshots use the same new model.
|
|
||||||
- Delete superseded loading and override machinery. Do not hide the old model
|
|
||||||
behind adapters that defeat the adoption's consistency benefit.
|
|
||||||
|
|
||||||
Keep Audita's domain capabilities, such as ordered and repeated correction modules,
|
|
||||||
expressible in the new model. Their existing configuration spelling is not binding.
|
|
||||||
Transcript/glossary formats, processing algorithms, and output schemas are outside
|
|
||||||
this migration unless later planning identifies a concrete dependency or an
|
|
||||||
explicitly coordinated change.
|
|
||||||
|
|
||||||
## Selected Overlays for Testing
|
|
||||||
|
|
||||||
Enable ConfigKit's optional selected-overlay feature in the migration. A testing
|
|
||||||
configuration that selects cheaper, faster LLMs is the initial use case. ConfigKit
|
|
||||||
**overlays** modify application settings; PromptKit **execution profiles** still
|
|
||||||
own model and generation settings. Keep `profiles_dir`, the PromptKit `profiles/`
|
|
||||||
directory, and proposal/validation profile names unchanged by this terminology
|
|
||||||
change.
|
|
||||||
|
|
||||||
The planned interface is `--overlay <name>` on `process`, `config validate`, and
|
|
||||||
`config print-effective`. Treat selection as invocation state passed to
|
|
||||||
`Options.Overlay`, not as another persistent setting or environment overlay.
|
|
||||||
Follow ConfigKit v0.4.0's selection contract: omission, an empty name, or `default`
|
|
||||||
selects no overlay; a named selection loads `overlays/<name>.yml` beneath the selected
|
|
||||||
configuration root. A missing selected overlay is an error. Use the library's
|
|
||||||
single-overlay semantics without adding stacking, inheritance, or legacy-directory
|
|
||||||
fallback.
|
|
||||||
|
|
||||||
Precedence is application defaults, main configuration, selected overlay, then
|
|
||||||
explicit setting flags. Perform Audita's role inheritance and semantic validation
|
|
||||||
on the resolved configuration. ConfigKit's base parsing and schema-version checks
|
|
||||||
still precede overlay application; overlays cannot replace the schema version or
|
|
||||||
contain ConfigKit directives. General environment-variable overrides remain retired.
|
|
||||||
Imports and secret-directory bootstrap remain disabled.
|
|
||||||
|
|
||||||
Provide a maintained `overlays/testing.yml` example that selects testing execution
|
|
||||||
profiles through `llm.proposal_profile` and `llm.validation_profile`. Model names and
|
|
||||||
generation controls belong in the corresponding PromptKit profile files. Set both
|
|
||||||
role selections in this example so a base configuration's explicit validation
|
|
||||||
profile cannot accidentally retain the production model. Normal omission-based
|
|
||||||
validation-profile inheritance continues to apply to the final effective settings.
|
|
||||||
The example must make its backend/credential requirements clear and use no real
|
|
||||||
secrets. Model choice is operator-controlled; “testing” does not promise a price,
|
|
||||||
performance level, or equivalent correction quality.
|
|
||||||
|
|
||||||
Overlays may modify other ordinary application settings supported by the schema;
|
|
||||||
they are not restricted to profile selection. The resulting configuration must
|
|
||||||
still satisfy shared backend capacity, credential, and domain rules. Validation,
|
|
||||||
effective-config inspection, selected-profile inspection, and processing must all
|
|
||||||
use the same selected overlay. Preserve overlay provenance in configuration
|
|
||||||
explanations and diagnostics while retaining secret redaction.
|
|
||||||
|
|
||||||
The [recorded overlay results](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
|
||||||
prove the representative load, inspection, and HTTP path. Production acceptance
|
|
||||||
requires the same selection to reach both actual LLM roles, with explicit flags
|
|
||||||
winning over overlays and surviving secret references resolved/redacted correctly.
|
|
||||||
The isolated prototype is evidence, not a second supported runtime.
|
|
||||||
|
|
||||||
## Inspection and Invocation Contract
|
|
||||||
|
|
||||||
`process`, `config validate`, and `config print-effective` share file/overlay
|
|
||||||
selection and retained setting overrides. Invocation I/O paths remain process-only.
|
|
||||||
Only explicitly supplied flags override settings. Validation requires an explicit
|
|
||||||
config file; processing and inspection use the selected discovery order. Help is
|
|
||||||
available before configuration loading. All three commands validate selected
|
|
||||||
profiles without requiring a provider call during setup.
|
|
||||||
|
|
||||||
`config print-effective --view settings|profiles|explain` exposes three YAML views:
|
|
||||||
|
|
||||||
- `settings` (default): ConfigKit's redacted effective configuration.
|
|
||||||
- `profiles`: an Audita-owned summary of each selected role's profile, backend,
|
|
||||||
model, temperature, token limit, top-p, timeout, service tier, and reasoning
|
|
||||||
effort where present. Arbitrary extra provider parameters and credentials are
|
|
||||||
omitted; this is a selected-controls view, not every execution input.
|
|
||||||
- `explain`: field paths, sources, and derivation records from ConfigKit, without
|
|
||||||
resolved secret values.
|
|
||||||
|
|
||||||
A ConfigKit snapshot alone does not describe profile-owned controls. Profile
|
|
||||||
inspection must enforce the registered-backend and credential boundary. Diagnostic
|
|
||||||
snapshots use these safe views and preserve overlay/explicit-override provenance.
|
|
||||||
The migration guide explains the intentional replacement of legacy inspection JSON.
|
|
||||||
|
|
||||||
## Integration and Completion Criteria
|
|
||||||
|
|
||||||
ConfigKit and PromptKit form one intentional breaking refactor. Intermediate
|
|
||||||
commits need not build or run. The target contains no temporary old-runtime adapter,
|
|
||||||
compatibility shim, processing gate, dual loader, or automatic legacy conversion.
|
|
||||||
The [combined feature roadmap](library-adoption.md) owns execution/reporting intent;
|
|
||||||
[implementation.md](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) alone owns stage order and implementation tasks.
|
|
||||||
|
|
||||||
The final application loads all retained domain settings, assembles both selected
|
|
||||||
profiles, and processes through PromptKit. Controlled HTTP evidence must connect
|
|
||||||
configuration selection to actual requests, domain outcomes, safe diagnostics,
|
|
||||||
and shared finite backend capacity. Maintained examples and one migration guide
|
|
||||||
cover the complete file/flag/environment transition. Obsolete application-owned
|
|
||||||
loading, transport, rendering, and scheduler machinery is removed.
|
|
||||||
|
|
||||||
The combined target is Audita v2.0.0. Release compliance and publication remain
|
|
||||||
last and separate from implementing the refactor. Retention and transcript timestamp
|
|
||||||
changes are not included; the selected retry/repair contract resolves the retry
|
|
||||||
follow-up in [runtime behavior](runtime-behavior.md).
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
- Pin ConfigKit v0.4.0 in production and align Go/toolchain requirements. ConfigKit requires
|
|
||||||
Go 1.25; the combined PromptKit prototype requires Go 1.25.5. Adopting these libraries
|
|
||||||
does not automatically remove Audita's existing YAML dependency used for glossary parsing.
|
|
||||||
- Implement the selected configuration and final PromptKit runtime directly,
|
|
||||||
without temporary runtime adapters, compatibility shims, or process gates.
|
|
||||||
- Enable selected overlays as specified above. Keep imports and secret-directory
|
|
||||||
bootstrap disabled unless a separate concrete requirement justifies them.
|
|
||||||
- Validation and inspection resolve surviving credential references and fail on
|
|
||||||
missing or empty values; no credential-free bypass is planned.
|
|
||||||
- Inspection and diagnostics use the safe selected-profile view and ConfigKit
|
|
||||||
provenance described above. Changed formats are covered by the migration guide.
|
|
||||||
|
|
||||||
The eventual completion test is that Audita follows the suite's ConfigKit
|
|
||||||
conventions while retaining its intended domain capabilities, with a clear
|
|
||||||
migration guide and no ongoing obligation to support the previous configuration
|
|
||||||
model.
|
|
||||||
@@ -1,162 +0,0 @@
|
|||||||
# Partial Success for Validator Decision Sets
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
|
||||||
|
|
||||||
**Status:** Implemented. This document owns the target behavior and user intent.
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md) owns concrete decisions, ordered work,
|
|
||||||
and verification.
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Preserve usable validator decisions when other entries in the same model response
|
|
||||||
have missing, duplicate, or unexpected correction indices. One defective entry
|
|
||||||
should not prevent unrelated candidates from completing their required validation.
|
|
||||||
This extends Audita's existing partial-success approach without another model call.
|
|
||||||
|
|
||||||
Implement this simpler improvement before reconsidering guided domain recovery.
|
|
||||||
Missing validation must never authorize applying a correction, but neither should
|
|
||||||
it invalidate a different candidate's unambiguous approval or editorial rejection.
|
|
||||||
The existing [resilience design](resilience.md) remains the foundation.
|
|
||||||
|
|
||||||
## Historical Baseline
|
|
||||||
|
|
||||||
The reviewed application baseline is `d2a6fcf`:
|
|
||||||
|
|
||||||
- PromptKit validates the complete model response against Audita's embedded JSON
|
|
||||||
Schema and performs its configured structural repairs. Only structurally valid,
|
|
||||||
decoded responses reach domain decision mapping.
|
|
||||||
- `mapLLMResponseToDecisions` in
|
|
||||||
[`llm_validators.go`](../../internal/framework/validators/llm_validators.go)
|
|
||||||
requires exactly one decision for every requested correction index. An empty
|
|
||||||
decision list, missing index, duplicate index, or unexpected index returns an
|
|
||||||
error instead of retaining usable decisions.
|
|
||||||
- `LLMBackedValidator.Validate` consequently marks the entire batch unavailable
|
|
||||||
with `validation_decisions_invalid`. Other batches continue, but usable decisions
|
|
||||||
within that batch are lost.
|
|
||||||
- The runner's cardinality checks enforce an internal contract: each candidate
|
|
||||||
receives one outcome from each validator it reaches. These checks apply to
|
|
||||||
Audita-owned results, not directly to raw model entries.
|
|
||||||
- Deterministic validators already filter individual proposals. Unpreviewable
|
|
||||||
candidates and oversized validation inputs are isolated; recoverable provider
|
|
||||||
failures remain batch-local; application skips individual conflicts.
|
|
||||||
- Ordinary reports distinguish failed operations from successful operations with
|
|
||||||
recoveries. Several failed operations in one stage/chunk share one work-unit
|
|
||||||
error record. The [output contract](../integrations/output-files.md) owns the
|
|
||||||
current counts and serialized representation.
|
|
||||||
|
|
||||||
The defect addressed here is the all-or-nothing domain mapping boundary, not
|
|
||||||
validation itself or the runner's internal consistency checks.
|
|
||||||
|
|
||||||
## Target Decision Semantics
|
|
||||||
|
|
||||||
Reconcile each structurally valid response against the indices actually requested
|
|
||||||
in that particular validator batch. Correction indices, rather than segment IDs
|
|
||||||
or positions in the response array, identify candidates. Several corrections may
|
|
||||||
legitimately target the same segment.
|
|
||||||
|
|
||||||
| Response condition | Target behavior |
|
|
||||||
| --- | --- |
|
|
||||||
| Exactly one valid decision for a requested index | Preserve its approval or rejection and associated decision evidence. |
|
|
||||||
| Requested index omitted | Produce an explicit validation-unavailable outcome for that candidate. |
|
|
||||||
| Requested index appears more than once | Treat that candidate's validation as unavailable; preserve unrelated decisions. |
|
|
||||||
| Index was not requested in this batch | Ignore the extraneous entry and record the handled defect. |
|
|
||||||
| Empty decision list for a nonempty request | Mark all requested candidates unavailable; do not fail the run. |
|
|
||||||
| Decisions appear in a different order | Match by identity and retain deterministic internal results. |
|
|
||||||
|
|
||||||
Duplicate handling does not choose the first or last answer, nor infer a verdict
|
|
||||||
by voting. Even identical duplicates follow the agreed duplicate-index handling;
|
|
||||||
accepting them specially is not needed for this feature. Duplicate extraneous
|
|
||||||
indices remain extraneous entries, not reasons to invalidate requested candidates.
|
|
||||||
|
|
||||||
An unavailable candidate remains unapplied and does not proceed as approved to
|
|
||||||
later validators. It is distinct from a valid `approved: false` decision, which
|
|
||||||
is ordinary editorial processing. Valid approvals continue through the remaining
|
|
||||||
required validators and apply-time checks; this feature grants no bypass.
|
|
||||||
|
|
||||||
The mapper returns one internal outcome for every requested candidate and no
|
|
||||||
outcome for an extraneous index. Preserve strict runner cardinality checks so
|
|
||||||
application bugs are still detected. Do not convert genuine internal invariant
|
|
||||||
failures, cancellation, or fatal setup failures into tolerated model defects.
|
|
||||||
|
|
||||||
## Outcomes and Reporting
|
|
||||||
|
|
||||||
Keep existing work-unit aggregation and operation accounting:
|
|
||||||
|
|
||||||
- If any requested candidate lacks an unambiguous usable decision, record a
|
|
||||||
validation-operation error identifying the affected requested indices. Other
|
|
||||||
candidates' usable decisions from that operation remain effective. An operation
|
|
||||||
error does not mean every candidate in its batch failed.
|
|
||||||
- If all requested candidates have usable decisions and extra entries were
|
|
||||||
ignored, record a successful operation with a recovery for that handled defect.
|
|
||||||
This remains a recovery even if all legitimate decisions are editorial rejections.
|
|
||||||
- If missing/duplicate requested decisions and ignored extras coexist, retain the
|
|
||||||
operation error. Follow the existing rule that failed operations do not also
|
|
||||||
produce recovery records; retain extra-entry details in debug evidence.
|
|
||||||
- A valid complete response with no handled defect adds neither error nor recovery.
|
|
||||||
Valid editorial rejection alone never adds either category.
|
|
||||||
- Preserve actual structural repair and service-retry counts. On successful
|
|
||||||
operations, consolidate applicable recovery reasons in one recovery record.
|
|
||||||
Do not count reconciliation as another LLM operation or structural repair.
|
|
||||||
|
|
||||||
Error attribution must distinguish affected candidates from the rest of the batch.
|
|
||||||
Debug evidence should preserve supplied decisions, ignored entries, and the reason
|
|
||||||
an expected candidate became unavailable. Do not fabricate candidate ledger entries
|
|
||||||
for unknown indices or mislabel missing decisions as deliberate editorial rejection.
|
|
||||||
Ordinary reports must retain safe metadata without raw transcript or response text.
|
|
||||||
|
|
||||||
Use a decision-specific recovery reason rather than treating ignored validator
|
|
||||||
entries as defective correction proposals. Concrete reason names and minimal data
|
|
||||||
model changes belong in the implementation plan. Keep YAML, Markdown, and CLI
|
|
||||||
counts consistent, with the existing reporting-disable and independent debug options.
|
|
||||||
|
|
||||||
## Ownership and Target Architecture
|
|
||||||
|
|
||||||
Audita's validator layer reconciles structurally valid model responses into
|
|
||||||
complete internal candidate outcomes. The runner preserves strict internal
|
|
||||||
cardinality and advances only approved candidates. Reporting consumes operation
|
|
||||||
facts without treating partial validation failure as loss of every decision.
|
|
||||||
Debug evidence distinguishes unavailable validation from editorial rejection.
|
|
||||||
|
|
||||||
The schema/transport boundary remains with PromptKit; reconciliation needs no
|
|
||||||
additional model call. Concrete helper structures, reason names, mapping changes,
|
|
||||||
and verification steps belong in the [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
|
||||||
|
|
||||||
## Scope Boundaries
|
|
||||||
|
|
||||||
PromptKit's whole-response structural validation and repair remain unchanged.
|
|
||||||
A malformed entry, including an extraneous entry that violates the schema, may
|
|
||||||
still invalidate the response and exhaust structural repairs before reconciliation
|
|
||||||
can run. Ignoring schema-valid extras does not promise salvage from malformed JSON,
|
|
||||||
missing required fields, wrong types, or out-of-range values.
|
|
||||||
|
|
||||||
General per-entry structural salvage is a separate design question, not part of
|
|
||||||
this fix. Likewise, guided regeneration remains deferred for reconsideration after
|
|
||||||
this change; no new retries, feedback prompts, budgets, upstream dependency changes,
|
|
||||||
or operator controls are required here.
|
|
||||||
|
|
||||||
Preserve module scope, literal replacement behavior, fixed chunks, stage barriers,
|
|
||||||
span-deletion safeguards, backend capacity limits, and fatal cancellation behavior.
|
|
||||||
No additional restrictions on valid corrections or guarantees about model judgment
|
|
||||||
are introduced. This work does not itself authorize tagging or publishing v2.0.0.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A schema-valid response can retain usable decisions despite missing, duplicate,
|
|
||||||
or unexpected indices elsewhere in the same batch.
|
|
||||||
- Only the affected requested candidates become unavailable; extras never create
|
|
||||||
candidates or attach decisions to a different batch's candidates.
|
|
||||||
- Valid negative decisions remain editorial outcomes, and every applied correction
|
|
||||||
completes all required validation.
|
|
||||||
- Internal validator results satisfy existing strict cardinality checks.
|
|
||||||
- Missing/duplicate requested decisions yield accurately attributed errors without
|
|
||||||
losing valid work. Extra-only defects yield a consolidated recovery; mixed
|
|
||||||
defects follow existing failed-operation reporting semantics.
|
|
||||||
- Final transcript, debug evidence, YAML reports, Markdown counts, and CLI summary
|
|
||||||
agree about applied work and failures/recoveries.
|
|
||||||
- Reconciliation makes no additional provider calls. Existing structural failures,
|
|
||||||
provider failures, and cancellation retain their documented handling.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
No unresolved product-policy questions block implementation. Guided recovery and
|
|
||||||
structural salvage remain separate future decisions.
|
|
||||||
@@ -1,19 +1,20 @@
|
|||||||
# Deferred Feature Work
|
# Deferred Feature Work
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
**Last updated:** 2026-11-15
|
||||||
|
|
||||||
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
**Status:** Deferred or exploratory work; not implemented or scheduled. These
|
||||||
entries preserve scope outside the [resilience roadmap](resilience.md) and
|
entries preserve work beyond completed feature and library upgrades.
|
||||||
[span-deletion consistency roadmap](span-deletion.md). Each entry states its
|
Each entry states its intent; inclusion does not commit us to
|
||||||
intent; inclusion does not commit us to implementing every capability.
|
implementing every capability.
|
||||||
|
|
||||||
## Domain-Aware Response Recovery
|
## Domain-Aware Response Recovery
|
||||||
|
|
||||||
The [partial-success decision reconciliation](decision-set-reconciliation.md)
|
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
|
||||||
preserves usable decisions without another model call. Guided recovery remains
|
preserves usable decisions without another model call. Revision of rejected
|
||||||
deferred pending evidence from failures that remain. General per-entry
|
correction proposals is implemented with substantive
|
||||||
salvage from structurally invalid responses is also a separate design question;
|
validator feedback and explicit proposal withdrawal; see the
|
||||||
the accepted reconciliation feature retains whole-response structural validation.
|
[pipeline guide](../internal/pipeline.md).
|
||||||
|
It is distinct from recovering an unavailable validator decision.
|
||||||
|
|
||||||
Consider bounded feedback and regeneration for model responses that satisfy the
|
Consider bounded feedback and regeneration for model responses that satisfy the
|
||||||
structural schema but leave requested decisions unavailable after reconciliation.
|
structural schema but leave requested decisions unavailable after reconciliation.
|
||||||
@@ -30,11 +31,15 @@ secrets, and how regeneration fits bounded budgets and operation deadlines. Reus
|
|||||||
shared library capabilities where appropriate; do not introduce an uncoordinated
|
shared library capabilities where appropriate; do not introduce an uncoordinated
|
||||||
outer retry loop that multiplies existing retry/repair budgets.
|
outer retry loop that multiplies existing retry/repair budgets.
|
||||||
|
|
||||||
Ordinary editorial rejection is a valid decision, not a reason to regenerate until
|
Editorial rejection remains a valid decision about the evaluated proposal. The
|
||||||
an edit is approved. Recovery must preserve required validation, candidate identity,
|
implemented correction repair loop handles subsequent revision and full
|
||||||
partial-success safety, and truthful error/recovery reporting. Validate with
|
revalidation; this deferred entry concerns unavailable decisions. Decision recovery
|
||||||
controlled responses that distinguish recoverable protocol defects from valid
|
must preserve required validation, candidate identity, partial-success safety,
|
||||||
negative decisions. Exact APIs, budgets, and sequencing remain undecided.
|
and truthful error/recovery reporting. Validate with controlled responses that
|
||||||
|
distinguish protocol defects from valid negative decisions. Exact APIs, budgets,
|
||||||
|
and sequencing remain undecided. General per-entry salvage from structurally
|
||||||
|
invalid responses also remains a separate design question; current reconciliation
|
||||||
|
retains whole-response structural validation.
|
||||||
|
|
||||||
## Independent Whole-Pipeline Traversal by Chunk
|
## Independent Whole-Pipeline Traversal by Chunk
|
||||||
|
|
||||||
@@ -59,7 +64,7 @@ No new scheduler, global limiter, or execution mode is selected by this entry.
|
|||||||
|
|
||||||
Potential future work, subject first to evaluating whether it is worth implementing
|
Potential future work, subject first to evaluating whether it is worth implementing
|
||||||
at all. Empty replacement text and deletion of a span within a surviving segment
|
at all. Empty replacement text and deletion of a span within a surviving segment
|
||||||
are covered by the accepted [span-deletion consistency roadmap](span-deletion.md).
|
are covered by the implemented [span-deletion behavior](../internal/modules.md#proposal-generation-ownership).
|
||||||
That feature does not remove segment records or permit blank resulting segments.
|
That feature does not remove segment records or permit blank resulting segments.
|
||||||
|
|
||||||
Before designing segment removal, review representative production examples to
|
Before designing segment removal, review representative production examples to
|
||||||
@@ -78,3 +83,16 @@ semantics. It must address meaning preservation and the effects on segment
|
|||||||
identity, timestamps, fixed chunks, subsequent stages, recombination, and output
|
identity, timestamps, fixed chunks, subsequent stages, recombination, and output
|
||||||
consumers. Do not infer authorization to remove a segment from an empty string
|
consumers. Do not infer authorization to remove a segment from an empty string
|
||||||
or introduce segment-removal machinery as part of span-deletion consistency.
|
or introduce segment-removal machinery as part of span-deletion consistency.
|
||||||
|
|
||||||
|
## Live-Model Editorial Evaluation
|
||||||
|
|
||||||
|
Consider an opt-in transcript corpus and live-model comparisons to assess needed
|
||||||
|
and unnecessary edits, missed corrections, validator disagreement, and repair
|
||||||
|
outcomes through independent editorial review. Native PromptKit execution
|
||||||
|
summaries can supply request and usage measurements. Keep live calls outside
|
||||||
|
the offline test suite.
|
||||||
|
|
||||||
|
Broader validator context and combined semantic validation remain exploratory.
|
||||||
|
A proposal to expand context should identify a captured case and the evidence
|
||||||
|
missing from the validator request.
|
||||||
|
See the [current validator context](../internal/validators.md#supplemental-validation-context).
|
||||||
|
|||||||
@@ -1,313 +0,0 @@
|
|||||||
# Implementation Plan: Standard Policy Alignment
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
|
||||||
|
|
||||||
**Status:** Complete; Stages 1–4 implemented.
|
|
||||||
**Audience:** gpt-5.6-terra coding agent.
|
|
||||||
**Audita planning baseline:** `91d9be68a7bb3a102900253fdfd8438f0a43f666`.
|
|
||||||
**Selected upstream revision:** `d84a38ebab26f048cafbe59a2246700c35d29843`.
|
|
||||||
|
|
||||||
## Scope and Execution
|
|
||||||
|
|
||||||
Implement the accepted [Standard Policy Alignment roadmap](policy.md). It owns
|
|
||||||
the intended end state and reasons for local variations; this plan owns ordered
|
|
||||||
implementation and verification. Complete Stage 1 through Stage 4 in order, each
|
|
||||||
as one focused agent task. Record completion evidence under each stage and commit
|
|
||||||
completed units with short, plain English messages under the agent policy.
|
|
||||||
|
|
||||||
Begin with [development guidance](../development.md) and the
|
|
||||||
[agent policy](../policy/agents.md). Read the policies governing each stage;
|
|
||||||
reuse unchanged policy context. For policy edits, read that policy and affected
|
|
||||||
cross-references. Follow the adopted selective reading guide once installed.
|
|
||||||
Use graph-first discovery for application code; document and shell/configuration
|
|
||||||
inspection does not need a graph lookup.
|
|
||||||
|
|
||||||
Use the public upstream repository at
|
|
||||||
`https://gitea.maximumdirect.net/eric/policy` and check out the exact revision above
|
|
||||||
in a temporary directory outside Audita. Verify its full commit before copying.
|
|
||||||
Read its `docs/adoption.md`, relevant `docs/policy/` files, `COPYRIGHT`, and
|
|
||||||
referenced assets. Do not replace the selected revision with current main or
|
|
||||||
reconstruct prior adoption baselines. The public repository and its attribution
|
|
||||||
are now explicitly approved.
|
|
||||||
|
|
||||||
This plan replaces the completed decision-set plan, preserved at
|
|
||||||
[the planning baseline](https://gitea.maximumdirect.net/eric/audita/src/commit/91d9be68a7bb3a102900253fdfd8438f0a43f666/docs/roadmap/implementation.md).
|
|
||||||
Repair historical roadmap references in Stage 4; do not reopen those features.
|
|
||||||
|
|
||||||
Implement directly toward the final state. Intermediate commits need no temporary
|
|
||||||
policy files, compatibility machinery, or complete dependency inventory. Keep
|
|
||||||
partial work truthful and finish cross-document consistency by Stage 4. Do not
|
|
||||||
release, tag, push, rewrite published material, change dependency versions, or
|
|
||||||
refactor application code as part of this work. No extra approval round is needed
|
|
||||||
between stages. If factual verification uncovers a concrete unresolved rights or
|
|
||||||
publication issue, report it rather than inventing an obligation or claiming it
|
|
||||||
resolved.
|
|
||||||
|
|
||||||
## Common Adoption Decisions
|
|
||||||
|
|
||||||
- Start each policy from the selected upstream file. Preserve shared substantive
|
|
||||||
wording verbatim. Remove its `Base Policy Template` section after fulfilling
|
|
||||||
the instructions; include only the project customizations selected below.
|
|
||||||
- Retain the upstream base version and set `Last Updated` to the actual adoption
|
|
||||||
date. Add `**Upstream Source:**` with a Markdown link to that same file at the
|
|
||||||
full selected commit, using the URL prefix
|
|
||||||
`https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/`.
|
|
||||||
Do not claim this was the source of old local documents.
|
|
||||||
- Keep discovery URLs in `meta.md` pointed to the public repository and
|
|
||||||
`/src/branch/main/docs/adoption.md`. Pin example links in the adopted OSS policy
|
|
||||||
to this selected commit, after checking their paths. Preserve examples upstream.
|
|
||||||
- Explain substantive local variations briefly where they apply. No separate
|
|
||||||
exception register, synchronization system, policy snapshot tests, or legal
|
|
||||||
approval workflow is needed.
|
|
||||||
- Keep policy credit in the single canonical attribution entry. Describe only
|
|
||||||
material adopted so far, expanding it as later stages copy more material.
|
|
||||||
Metadata and template removal mean an adopted file can truthfully be described
|
|
||||||
as adapted even when its substantive rules are verbatim.
|
|
||||||
- Ordinary link/text review is sufficient for documentation changes. Do not run
|
|
||||||
application tests simply because a policy document changed. Each stage ends
|
|
||||||
with `git diff --check` and review of its staged diff before committing.
|
|
||||||
|
|
||||||
## Stage 1 — Establish Provenance and Shared Agent/Testing Policies
|
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Adopted meta 1.0, agents 1.3, and testing 1.3 from
|
|
||||||
the verified upstream commit, preserving Audita's graph-first discovery rule.
|
|
||||||
The initial attribution and scoped copyright notice identify only these three
|
|
||||||
adapted policies; the CC-BY-SA text matches upstream byte-for-byte. Substantive
|
|
||||||
policy text, pinned source paths, local links, and `git diff --check` were checked.
|
|
||||||
|
|
||||||
**Files:** `docs/policy/meta.md`, `docs/policy/agents.md`,
|
|
||||||
`docs/policy/testing.md`, `docs/oss/attribution.md`,
|
|
||||||
`docs/oss/licenses/CC-BY-SA-4.0.txt`, and `COPYRIGHT`.
|
|
||||||
|
|
||||||
1. Adopt meta 1.0, agents 1.3, and testing 1.3 with the common metadata and
|
|
||||||
template-removal rules. Preserve the existing Codebase Discovery section as
|
|
||||||
the one local addition to agents: Audita deliberately uses graph-first
|
|
||||||
discovery. Adopt the remaining agent rules and the testing rules verbatim.
|
|
||||||
2. Create attribution with `# Open Source Attribution Notices`, followed directly
|
|
||||||
by the canonical entry from `examples/licensing/policy-attribution.md`.
|
|
||||||
Substitute the selected commit in its COPYRIGHT, LICENSE, and Source URLs.
|
|
||||||
Initially identify the three newly adapted policy files in its `Material:`
|
|
||||||
sentence. Do not claim the remaining files have already been updated.
|
|
||||||
3. Copy the selected revision's `docs/oss/licenses/CC-BY-SA-4.0.txt` byte-for-byte.
|
|
||||||
Keep the supplied upstream copyright and links to its grant/disclaimer.
|
|
||||||
4. Add the adoption guide's scoped CC-BY-SA notice to Audita's `COPYRIGHT`, with
|
|
||||||
affected material identified through attribution. Preserve Audita's name,
|
|
||||||
copyright, GPL-3.0-or-later application grant, warranty statement, and third-party
|
|
||||||
statement. Leave the root GPLv3 `LICENSE` unchanged.
|
|
||||||
5. Keep metadata local and truthful. `meta.md` points to the local attribution
|
|
||||||
and developer guide even though selective reading guidance is added next.
|
|
||||||
|
|
||||||
**Validation and completion:** Compare the substantive agent/testing text directly
|
|
||||||
with the selected upstream documents, allowing only template removal, metadata,
|
|
||||||
and the identified discovery addition. Verify each new source link's exact file
|
|
||||||
and revision, local attribution links, copied license bytes, and narrow grant
|
|
||||||
scope. No application test or publication is required.
|
|
||||||
|
|
||||||
## Stage 2 — Customize Remaining Policies and Install Selective Orientation
|
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Adopted architecture 1.3, documentation 1.11,
|
|
||||||
release 1.4, and OSS 1.8 with the selected Audita variations. Installed the
|
|
||||||
upstream root orientation and verbatim Policy Reading Guide, expanded the
|
|
||||||
scoped attribution, and repaired the release documentation heading link.
|
|
||||||
Checked versions, provenance and example destinations, local links, policy
|
|
||||||
differences, and `git diff --check`.
|
|
||||||
|
|
||||||
**Files:** `docs/policy/architecture.md`, `docs/policy/documentation.md`,
|
|
||||||
`docs/policy/release.md`, `docs/policy/oss.md`, `AGENTS.md`,
|
|
||||||
`docs/development.md`, `docs/oss/attribution.md`, and `COPYRIGHT` as needed for
|
|
||||||
accurate copied-guidance scope.
|
|
||||||
|
|
||||||
1. Adopt architecture 1.3. Retain the upstream general Purpose And Ownership,
|
|
||||||
Architectural Approach, and Adopted Constraints And Implemented Structure
|
|
||||||
guidance. Replace Application-Specific Topics with Audita's established
|
|
||||||
purpose, ownership/dependency boundaries, correction/concurrency rules,
|
|
||||||
recovery, state/compatibility, secret handling, and selected library ownership.
|
|
||||||
Reuse existing local decisions; avoid duplicating general upstream prose.
|
|
||||||
Replace example threat discussion with the existing secret/input boundaries
|
|
||||||
and canonical links where useful. Do not infer new threat exclusions,
|
|
||||||
filesystem guarantees, interfaces, or package restructuring from the template.
|
|
||||||
2. Adopt documentation 1.11. Keep applicable upstream ownership rows verbatim;
|
|
||||||
omit public HTTP and public Go API rows. Add the existing concrete ownership
|
|
||||||
of `docs/integrations/output-files.md` and contributor change recipes, and
|
|
||||||
retain the deliberate retirement of standalone troubleshooting documentation.
|
|
||||||
Account for the retained agent discovery/commit responsibilities in its owner
|
|
||||||
row. Keep reading triggers in development guidance, not a second policy table.
|
|
||||||
3. Adopt release 1.4. Retain upstream completion, validation, guard, immutability,
|
|
||||||
recovery, and preparation language. Preserve the existing Supported Platforms
|
|
||||||
table and unsupported-platform statement as Audita's support commitment.
|
|
||||||
Replace the upstream discretionary human-gate paragraph with Audita's selected
|
|
||||||
rule against manual approvals, sign-offs, and routine confirmations as release
|
|
||||||
gates. Remove the obsolete list describing now-shared upstream rules as local
|
|
||||||
deviations. Keep concrete atomic-push mechanics in the runbook.
|
|
||||||
4. Adopt OSS 1.8. Add a concise Audita context paragraph identifying the existing
|
|
||||||
GPL-3.0-or-later grant and linking to COPYRIGHT and the release procedure for
|
|
||||||
Gitea distribution. Use the upstream House Format and interpretation language
|
|
||||||
verbatim. Pin application/library COPYRIGHT and attribution examples to the
|
|
||||||
selected upstream commit; remove template-only setup text. No license change
|
|
||||||
to application code or blanket dependency compatibility claim is authorized.
|
|
||||||
5. Replace root orientation with upstream `agents/AGENTS.md` wording while
|
|
||||||
retaining any existing applicable repository instructions. Keep detailed
|
|
||||||
discovery rules in their current agent-policy owner rather than duplicating
|
|
||||||
them in a newly created block.
|
|
||||||
6. Insert the adoption guide's Policy Reading Guide and accompanying reading rules
|
|
||||||
into `docs/development.md` verbatim. Remove the blanket architecture-reading
|
|
||||||
instruction; preserve contributor commands and component-specific task rows.
|
|
||||||
Adjust cross-links that would otherwise contradict selective reading.
|
|
||||||
7. Expand the single policy attribution entry to identify all `docs/policy/`
|
|
||||||
documents, adapted root orientation, and the copied Policy Reading Guide in
|
|
||||||
`docs/development.md`. Scope the COPYRIGHT CC-BY-SA notice to those materials,
|
|
||||||
not unrelated contributor prose or application documentation. Keep the material
|
|
||||||
sentence factual and concise; retain the pinned upstream credit links.
|
|
||||||
8. Find and repair local links to policy headings renamed by adoption, including
|
|
||||||
the release correction heading. Do not edit published historical release
|
|
||||||
notes to retrofit new headings or licensing conventions.
|
|
||||||
|
|
||||||
**Validation and completion:** Compare all seven policies to the pinned source;
|
|
||||||
review every substantive difference against the roadmap's justified variations.
|
|
||||||
Check versions (1.3, 1.3, 1.11, 1.3, 1.4, 1.0, 1.8 for agents, architecture,
|
|
||||||
documentation, testing, release, meta, OSS respectively), file-specific source
|
|
||||||
links, removed template sections, example destinations, ownership consistency,
|
|
||||||
and selective reading. Temporary pending release/attribution implementation is
|
|
||||||
completed by the following stages; do not create scaffolding for it.
|
|
||||||
|
|
||||||
## Stage 3 — Complete Component Attribution and README Navigation
|
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Both CGO-disabled Linux target dependency lists
|
|
||||||
selected the same eight production modules. Inspected their selected source
|
|
||||||
notices and copied assets, excluded test-only dependencies, and resolved the
|
|
||||||
Go 1.25.5 source tag and its four incorporated vendored modules. Added 14
|
|
||||||
House Format entries including policy provenance, complete applicable license
|
|
||||||
texts, and the README footer. Checked 33 pinned source and notice destinations,
|
|
||||||
license bytes, local links, entry shape, and `git diff --check`.
|
|
||||||
|
|
||||||
**Files:** `docs/oss/attribution.md`, applicable texts under
|
|
||||||
`docs/oss/licenses/`, and `README.md`. Add factual exception treatment to the OSS
|
|
||||||
policy only if the actual inventory uncovers a concrete need.
|
|
||||||
|
|
||||||
1. Inventory actual incorporated material using the selected modules, production
|
|
||||||
package dependencies for both Linux targets, source/assets, and shipped Go
|
|
||||||
runtime. Use `go list -deps` for `./cmd/audita` with `CGO_ENABLED=0` and each
|
|
||||||
target architecture, plus source inspection for copied material. Inspect test
|
|
||||||
dependencies where relevant to included material; a go.mod entry or tool use
|
|
||||||
alone is not sufficient reason for attribution. Do not change go.mod/go.sum.
|
|
||||||
2. At the planning baseline, production dependencies include ConfigKit v0.4.1,
|
|
||||||
PromptKit v0.11.1, both catalogs v2.20260926.0, jsonschema/v6 v6.0.2,
|
|
||||||
yaml/v4 v4.0.0-rc.6, x/text v0.14.0, and yaml.v3 v3.0.1. Start with the four
|
|
||||||
verified source commits recorded in the feature roadmap; resolve remaining
|
|
||||||
module versions to exact source commits and inspect their COPYRIGHT, LICENSE,
|
|
||||||
NOTICE, PATENTS, and relevant bundled material as applicable. Account for
|
|
||||||
additional material discovered in that inspection rather than treating this
|
|
||||||
starting list as an exhaustive guarantee.
|
|
||||||
3. For the Go runtime, use the release build's configured Go 1.25.5 source tag
|
|
||||||
and resolve it to its full source commit. Do not silently substitute the
|
|
||||||
inspecting agent's installed toolchain (Go 1.26.8 during planning). Preserve
|
|
||||||
actual runtime/standard-library notices, including relevant bundled third-party
|
|
||||||
material. Document maintenance of this entry when the distributed toolchain
|
|
||||||
changes; do not pin new toolchain behavior or reject other supported local Go
|
|
||||||
versions merely to simplify attribution.
|
|
||||||
4. Add entries following OSS 1.8 exactly, with selected module/version or material
|
|
||||||
scope where useful. Preserve supplied copyrights and accurate license
|
|
||||||
expressions; use the prescribed link labels and `[Source](COMMIT_URL).`
|
|
||||||
spelling. Keep policy attribution first. Order other entries consistently by
|
|
||||||
project name; ordering is presentation, not a new public contract.
|
|
||||||
5. Copy applicable complete standard texts from verified sources, using the
|
|
||||||
pinned reference collection where applicable. Share identical standard texts
|
|
||||||
and include required companions (LGPLv3 and GPLv3 for the two libraries).
|
|
||||||
Include upstream commit links when a file contains project-specific material
|
|
||||||
in addition to standard wording. Preserve existing incorporated notices.
|
|
||||||
6. Apply the adopted house interpretation for project-specific notice links.
|
|
||||||
Put concrete exception reasoning in OSS policy or an appropriate decision
|
|
||||||
record, never in attribution. Do not add missing-copyright filler, research
|
|
||||||
narratives, inferred licensing choices, or legal conclusions to entries.
|
|
||||||
7. End README with the two OSS statements, identifying Audita as GPL v3.0 or later
|
|
||||||
and linking to `COPYRIGHT` and `docs/oss/attribution.md`. The linked grant owns
|
|
||||||
the separately licensed policy scope; no duplicated notice list is needed.
|
|
||||||
|
|
||||||
**Validation and completion:** Inspect entries against selected source evidence,
|
|
||||||
resolve every source/notice commit and path, compare standard license bytes, and
|
|
||||||
check local links. Confirm that actual distributed material is accounted for
|
|
||||||
without blindly including dependency graph or development-tool entries. Record
|
|
||||||
concise inventory/check evidence under this stage; no new SBOM, scanner, durable
|
|
||||||
inventory generator, or test suite is required.
|
|
||||||
|
|
||||||
## Stage 4 — Integrate Release Notes and Finish Cross-Document Validation
|
|
||||||
|
|
||||||
**Completed:** 2026-09-28. Wired the checked-in release note into the hosted
|
|
||||||
release plugin, documented immutable-tag notice links and release preparation,
|
|
||||||
repaired the specified historical roadmap references, and marked the feature
|
|
||||||
roadmap implemented. Parsed the YAML, substituted `v2.0.0` to a tracked note,
|
|
||||||
checked non-overwrite settings and candidate footer paths, reviewed all seven
|
|
||||||
policies against the pinned source, and checked repository-local Markdown links
|
|
||||||
and `git diff --check`. Hosted execution awaits the next real release.
|
|
||||||
|
|
||||||
**Files:** `.woodpecker/release.yml`, `docs/release.md`, affected documentation
|
|
||||||
links, `docs/roadmap/policy.md`, and this plan. Repair the historical references
|
|
||||||
specified below; do not retire unrelated roadmaps in this sprint.
|
|
||||||
|
|
||||||
1. Add this setting to the existing `publish-release` step:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
note: docs/releases/${CI_COMMIT_TAG}.md
|
|
||||||
```
|
|
||||||
|
|
||||||
The [official release-plugin documentation](https://woodpecker-ci.org/plugins/release)
|
|
||||||
supports a file path in `note`; the
|
|
||||||
[environment reference](https://woodpecker-ci.org/docs/usage/environment)
|
|
||||||
makes `CI_COMMIT_TAG` available for configuration substitution. These interfaces
|
|
||||||
were checked during planning. Use the checked-out tagged note directly; no
|
|
||||||
generator, custom publisher, or additional dependency is needed. Preserve
|
|
||||||
`overwrite: false`, `file-exists: skip`, existing assets/checksum settings,
|
|
||||||
secret reference, image, and step dependency.
|
|
||||||
2. In `docs/release.md`, add the footer preparation instruction with this pattern
|
|
||||||
(substitute the selected release version for `<version>`):
|
|
||||||
|
|
||||||
```markdown
|
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/<version>/docs/oss/attribution.md)
|
|
||||||
```
|
|
||||||
|
|
||||||
Explain that the plugin uses the checked-in note for new hosted releases and
|
|
||||||
preserves existing release information on retries. Keep old notes and pages
|
|
||||||
intact. Validate intended paths locally during preparation; do not add remote
|
|
||||||
footer checks, a footer parser, or a new release-script gate.
|
|
||||||
3. Align the runbook with the adopted policy: retain `origin`, `main`, atomic
|
|
||||||
source/tag publication, one offline validation pass, required Linux builds,
|
|
||||||
optional check mode, and asynchronous immutable delivery retries. Reference
|
|
||||||
the policy for requirements rather than introducing additional ones. Add
|
|
||||||
proportionate completed-roadmap retirement guidance for future release
|
|
||||||
preparation without making it a script-enforced condition. No release script
|
|
||||||
change is expected from this adoption.
|
|
||||||
4. Repair both current-plan links in `decision-set-reconciliation.md` to the
|
|
||||||
historical plan at the full Audita planning baseline listed above. Remove
|
|
||||||
the stale sentence in `library-adoption.md` saying the current implementation
|
|
||||||
plan covers span deletion. Preserve historical records and deferred work.
|
|
||||||
5. Review the final adoption diff against upstream and the feature roadmap.
|
|
||||||
Check all policy source metadata, template removal, local exception reasons,
|
|
||||||
copied-material scope, license text fidelity, local links/anchors, and pinned
|
|
||||||
upstream destinations. Ensure the feature roadmap records implemented status
|
|
||||||
only after its accepted outcomes are achieved.
|
|
||||||
6. Parse/review the affected YAML using an available YAML parser or Woodpecker
|
|
||||||
linter. Substitute a representative existing release tag locally and confirm
|
|
||||||
the note path resolves to the tracked note; old notes need not have the new
|
|
||||||
footer. Check a temporary example of the new footer against candidate paths.
|
|
||||||
Verify non-overwriting publication settings remain unchanged. Do not call the
|
|
||||||
forge to create a test release, run a plugin with production credentials, or
|
|
||||||
wait for CI. Static configuration and documented plugin-interface validation
|
|
||||||
are sufficient for this wiring change; report that hosted execution awaits
|
|
||||||
the next real release rather than claiming it was exercised.
|
|
||||||
|
|
||||||
**Validation and completion:** Run `git diff --check`, review staged content and
|
|
||||||
all affected links, and record the final verification evidence. No full Go suite
|
|
||||||
or build matrix is required when only documents and the note setting changed.
|
|
||||||
If an actual release-script behavior change becomes necessary, justify its
|
|
||||||
scope and run focused `go test ./scripts` coverage for that behavior rather than
|
|
||||||
inventing additional release gates. Commit the completed work; leave a clean
|
|
||||||
workspace without publishing.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None requiring a policy or design decision before implementation. The upstream
|
|
||||||
revision, local exceptions, attribution arrangement, target documents, and
|
|
||||||
release-body mechanism are selected. Exact component notice details and source
|
|
||||||
revisions are factual verification work in Stage 3, not permission gates. If
|
|
||||||
that inspection reveals an uncovered obligation, follow the adopted exception
|
|
||||||
procedure and identify the concrete decision instead of silently expanding scope.
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
# Coordinated ConfigKit and PromptKit Adoption
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
|
||||||
|
|
||||||
**Status:** Production implementation complete. ConfigKit v0.4.2 and PromptKit
|
|
||||||
v0.11.2 are adopted; [historical implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records the
|
|
||||||
completed checks. Release compliance and publication remain separate work.
|
|
||||||
|
|
||||||
## Goal and Ownership
|
|
||||||
|
|
||||||
Adopt ConfigKit and PromptKit to align Audita with the application suite's shared
|
|
||||||
configuration and LLM execution conventions. Consistency is the primary benefit;
|
|
||||||
removing overlapping application infrastructure is a secondary benefit. Preserve
|
|
||||||
Audita's transcript-correction capabilities and domain protections.
|
|
||||||
|
|
||||||
| Owner | Responsibilities |
|
|
||||||
| --- | --- |
|
|
||||||
| ConfigKit | Strict application configuration loading, presence/default semantics, explicit overrides, recorded resolution, secret references, and safe inspection |
|
|
||||||
| PromptKit | Prompt/profile formats and rendering, provider calls, structural output validation and repair, transient retries, engine-local backend admission, and execution observations |
|
|
||||||
| Audita | Setting meanings, file/CLI selection, role/profile association, normalization/chunking, ordered modules, domain validation, correction application, reports, diagnostic redaction, and artifact lifecycle |
|
|
||||||
|
|
||||||
The [configuration roadmap](configkit.md) owns configuration and inspection policy.
|
|
||||||
The [boundary findings](library-boundary-findings.md) own the selected defaults,
|
|
||||||
full migration inventory, failure mapping, and prototype evidence. The selected
|
|
||||||
production versions are ConfigKit v0.4.2 and PromptKit v0.11.2, with Go 1.25.5 as
|
|
||||||
the minimum. Use published modules, never sibling checkouts or local replacements.
|
|
||||||
The existing YAML dependency remains while glossary parsing needs it.
|
|
||||||
|
|
||||||
## Migration Posture
|
|
||||||
|
|
||||||
This is an intentional breaking transition targeting Audita v2.0.0. No backward
|
|
||||||
compatibility is required for existing installations, config files, CLI flags,
|
|
||||||
environment overlays, or inspection output. Provide one migration guide, without
|
|
||||||
legacy loaders, aliases, dual modes, or automatic conversion tooling.
|
|
||||||
|
|
||||||
Both libraries belong to one refactor. Intermediate commits need not build or run;
|
|
||||||
there is no supported intermediate product state. No temporary adapters, compatibility
|
|
||||||
shims, or processing gates are needed. The complete application must work and pass
|
|
||||||
its checks before the refactor is finished. Release compliance and publication
|
|
||||||
come last, with no intermediate public release.
|
|
||||||
|
|
||||||
## Configuration and Execution Target
|
|
||||||
|
|
||||||
ConfigKit selected overlays enable a testing configuration that chooses cheaper or
|
|
||||||
faster models. The example selects both proposal and validation profiles. ConfigKit
|
|
||||||
application overlays and PromptKit execution profiles remain separate concepts.
|
|
||||||
Complete operator-maintained profiles own model/generation controls; embedded
|
|
||||||
application prompts and schemas are not operator-selected files. No catalog wiring,
|
|
||||||
profile-inheritance feature, ConfigKit imports, or secret-directory bootstrap is
|
|
||||||
part of this migration.
|
|
||||||
|
|
||||||
Capacity controls serve two needs: respecting a local service's hard simultaneous-
|
|
||||||
query limit and bounding concurrent paid API activity. One finite limit per backend
|
|
||||||
is shared across both roles and every initial, retry, and repair attempt. Limits
|
|
||||||
are engine-local, not coordinated across processes. No additional global or
|
|
||||||
per-role limiter or fine-grained scheduling policy is required. Profiles must use
|
|
||||||
registered backends and ConfigKit-resolved credentials; endpoint-only bypass and
|
|
||||||
competing profile credential lookup are excluded.
|
|
||||||
|
|
||||||
Structural repair is enabled by default for both roles because additional attempts
|
|
||||||
to obtain valid output are desired. Transient retries and structural repairs have
|
|
||||||
separate budgets; explicit zero disables the respective behavior. Each generation,
|
|
||||||
including a repair, receives its own transient-retry budget. The Audita logical-call
|
|
||||||
deadline covers preparation, waiting, backoff, all generations, and repairs; profile
|
|
||||||
HTTP timeouts apply per attempt. Exact defaults and bounds are in the
|
|
||||||
[selected contract](library-boundary-findings.md#configuration-and-profile-contract).
|
|
||||||
No Audita outer retry loop restores the old broader retry policy.
|
|
||||||
|
|
||||||
Audita retains all four correction modules, five LLM-validator prompts,
|
|
||||||
deterministic validators, confidence/applicability checks, glossary protection,
|
|
||||||
validation batching/cardinality, deterministic proposal and ledger ordering,
|
|
||||||
transcript encoders, and artifact retention. PromptKit-valid JSON still passes
|
|
||||||
Audita's domain checks. Repair exhaustion, provider failure, and domain rejection
|
|
||||||
follow the [selected failure table](library-boundary-findings.md#execution-and-failure-mapping).
|
|
||||||
Retention and transcript timestamp changes remain separate work; this migration
|
|
||||||
resolves the [retry-count follow-up](runtime-behavior.md).
|
|
||||||
|
|
||||||
## Reporting and Diagnostic Target
|
|
||||||
|
|
||||||
Keep stable Audita-owned serialized records; do not expose raw library structs.
|
|
||||||
Use process report schema `v2` with the existing domain outcomes and a new
|
|
||||||
`llm_summary` separating logical calls, dispatched provider attempts, transient
|
|
||||||
retries, structural repairs, and usage. Transcript schema versions remain unchanged.
|
|
||||||
The implementation plan specifies the concrete new fields and artifact wiring.
|
|
||||||
|
|
||||||
Usage distinguishes known zero, unknown, and incomplete totals. Preserve observations
|
|
||||||
from failed calls even when no final result exists. Count terminal totals once;
|
|
||||||
attempt observations provide diagnosis without being added to those totals again.
|
|
||||||
This is operational evidence, not a complete billing ledger.
|
|
||||||
|
|
||||||
Replace global/role scheduler utilization with per-backend configured capacity and
|
|
||||||
observed queue/generation/retry/logical-call timing. Do not fabricate measurements
|
|
||||||
that the event API does not provide. Keep domain timing and deterministic correction
|
|
||||||
ledger ordering. Record prompt definition/rendered hashes and selected profile/model
|
|
||||||
without promising the old hash values. Callbacks are short, concurrency-safe, and
|
|
||||||
free of blocking I/O or waits on their own execution.
|
|
||||||
|
|
||||||
Effective configuration and selected-profile snapshots remain distinct safe views.
|
|
||||||
Secret values must not escape through inspection, logs, reports, runtime errors,
|
|
||||||
or retained diagnostic files. Observation records exclude payloads, URLs, raw
|
|
||||||
provider messages, prompts, and credentials. Existing artifact lifecycle and
|
|
||||||
partial-failure handling remain Audita responsibilities.
|
|
||||||
|
|
||||||
## Completion Criteria and Release Boundary
|
|
||||||
|
|
||||||
- Both libraries own their selected responsibilities in the production application.
|
|
||||||
Superseded loaders, transport, renderers, schedulers, aliases, and the isolated
|
|
||||||
prototype are gone once its useful evidence is represented in production tests.
|
|
||||||
- Offline tests prove configuration-to-execution routing for both roles, selected
|
|
||||||
overlays and explicit overrides, shared backend capacity, retry/repair outcomes,
|
|
||||||
cancellation/deadlines, domain decisions, redaction, and truthful partial reports.
|
|
||||||
Tests exercise real internal collaborators and controlled HTTP boundaries.
|
|
||||||
- Maintained examples and canonical docs describe the implemented final model.
|
|
||||||
One migration guide covers files, flags, environment inputs, profiles, credentials,
|
|
||||||
overlays, inspection, and changed runtime/report/diagnostic semantics.
|
|
||||||
- The assembled application passes its required checks; Linux amd64 and arm64 build.
|
|
||||||
macOS arm64 remains best effort. No live-provider or CI-success gate is added.
|
|
||||||
|
|
||||||
PromptKit v0.11.0's reviewed retry, observation, and capacity APIs remove the earlier
|
|
||||||
upstream feature prerequisites; no upstream edit is required for adoption. The
|
|
||||||
prototype proved representative behavior, not release compliance or model quality.
|
|
||||||
|
|
||||||
The selected PromptKit release uses LGPL-3.0-or-later. A later release unit must
|
|
||||||
review the actual production dependency graph and distribution requirements,
|
|
||||||
including notices and source/rebuild materials, before binary publication. This
|
|
||||||
implementation plan does not claim that review is complete. Release publication
|
|
||||||
uses the existing [procedure](../release.md) and completes on successful source/tag
|
|
||||||
push, without waiting for CI or artifact delivery.
|
|
||||||
@@ -1,392 +0,0 @@
|
|||||||
# Shared Library Boundary: Decisions and Evidence
|
|
||||||
|
|
||||||
**Status:** Boundary and offline prototype completed, 2026-09-23. Selected prototype contract and proposed
|
|
||||||
production migration target; not the current Audita configuration contract.
|
|
||||||
The [bounded roadmap](library-boundary.md) owns scope and completion criteria.
|
|
||||||
|
|
||||||
**Subsequent scope decision:** The [ConfigKit overlay plan](configkit.md#selected-overlays-for-testing)
|
|
||||||
adds optional selected overlays, with v0.4.0 as the migration baseline. It supersedes
|
|
||||||
the original no-overlay decision and adds an input layer between the main file and
|
|
||||||
explicit setting overrides. The v0.3.1 prototype results below remain historical
|
|
||||||
evidence; they do not establish overlay integration.
|
|
||||||
|
|
||||||
## Selected Boundary
|
|
||||||
|
|
||||||
The former prototype is retained in Git history at `590604b`; it was a separate
|
|
||||||
Go module using published ConfigKit v0.4.0, PromptKit v0.11.0, and
|
|
||||||
Audita v1.1.0. The Audita dependency lets this consumer reuse actual internal domain
|
|
||||||
helpers through its Audita module namespace without a local replacement. It does
|
|
||||||
not make those internal packages a new public API. Remove this dependency along
|
|
||||||
with the prototype after equivalent production integration tests landed.
|
|
||||||
|
|
||||||
The initial prototype results below were recorded with ConfigKit v0.3.1. On
|
|
||||||
2026-09-24, upgrading only its ConfigKit pin and sums to v0.4.0 required no source
|
|
||||||
changes. With Go 1.26.8 on linux/amd64, the nested offline `go test ./...` passed
|
|
||||||
and `go run .` still reported one accepted correction, four attempts, zero
|
|
||||||
transient retries, and two structural repairs. This verifies the dependency
|
|
||||||
upgrade, not selected-overlay behavior.
|
|
||||||
|
|
||||||
ConfigKit owns loading, presence, explicit overrides, secret resolution, and
|
|
||||||
redacted effective settings. PromptKit owns profile parsing, rendering, provider
|
|
||||||
execution, transient retries, structural repair, and backend admission. Audita
|
|
||||||
owns file selection, setting meanings, role selection, domain acceptance, and
|
|
||||||
mapping execution facts into reports. Production CLI and runtime remain unchanged.
|
|
||||||
|
|
||||||
## Configuration and Profile Contract
|
|
||||||
|
|
||||||
Selected for the prototype and recommended for production:
|
|
||||||
|
|
||||||
- Main YAML requires integer `version: 2`; absent, null, unknown fields, and older
|
|
||||||
versions fail. No legacy parser, aliases, environment overlays, or automatic
|
|
||||||
conversion. Defaults precede the selected file; explicitly supplied CLI settings
|
|
||||||
become ConfigKit overrides. Validation runs on the resolved result, so an override
|
|
||||||
may repair an invalid lower-precedence setting when decoding succeeds.
|
|
||||||
- Production file selection: explicit `--config`, then
|
|
||||||
`/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`. Select one file;
|
|
||||||
do not merge them. Missing system files are skipped; explicit missing or empty
|
|
||||||
paths and inaccessible selected files fail. No selected file fails with a request
|
|
||||||
to configure backend/profile selection: no hidden provider or model default.
|
|
||||||
Remove `AUDITA_CONFIG`. The prototype loads its explicit fixture only; discovery
|
|
||||||
and command parsing belong to the ConfigKit implementation stage.
|
|
||||||
- `profiles_dir` defaults to `profiles`, resolved relative to the selected main
|
|
||||||
configuration file, with absolute paths accepted. PromptKit reads that directory.
|
|
||||||
Prompts and schemas are application-embedded, not operator-selected prompt files.
|
|
||||||
The prototype uses complete application-owned profile files. Catalog profiles and
|
|
||||||
profile inheritance are not needed for this migration; do not add catalog wiring.
|
|
||||||
- `backends.<id>.endpoint` is required. Use one application backend identity per
|
|
||||||
service capacity pool. Do not split proposal and validation into different backend
|
|
||||||
IDs when they use the same service. Limits are engine-local, not shared across
|
|
||||||
multiple Audita processes or other applications.
|
|
||||||
- `backends.<id>.concurrency` defaults to **1**, must be positive, and is shared by
|
|
||||||
both roles and all initial/retry/repair attempts. Operators can select **4** for
|
|
||||||
the described local backend or a suitable finite paid API limit. There is no
|
|
||||||
additional global or per-role limiter. `queue_capacity` defaults to **1024**,
|
|
||||||
following PromptKit; non-negative, with zero meaning no waiting admission.
|
|
||||||
This is an operational escape hatch rather than a second scheduling hierarchy.
|
|
||||||
- `backends.<id>.credential` is an optional ConfigKit secret reference: a scalar
|
|
||||||
environment variable name. Omission or null means no credential. A supplied
|
|
||||||
reference must resolve to a nonempty value, including during validation and
|
|
||||||
inspection. There is no direct key field or CLI key flag. ConfigKit resolves it
|
|
||||||
once and Audita supplies `RunRequest.APIKey`. Profiles must not introduce another
|
|
||||||
`api_key_env` lookup. Backend credentials also support unauthenticated local use.
|
|
||||||
- Profiles own model, temperature, output token limit, top-p, per-generation timeout,
|
|
||||||
service tier, reasoning effort, and extra provider parameters using PromptKit's
|
|
||||||
format. They select the application `backend`; their effective endpoint must
|
|
||||||
equal that backend's endpoint. Endpoint-only execution is rejected because it
|
|
||||||
bypasses the requested finite capacity boundary. A named registered backend is
|
|
||||||
mandatory even if both roles use the same profile.
|
|
||||||
- `llm.proposal_profile` is required. Omitted `llm.validation_profile` inherits the
|
|
||||||
final proposal profile as a whole, with ConfigKit recording the derivation.
|
|
||||||
Explicit empty or null values fail; no individual model/endpoint/key inheritance.
|
|
||||||
- `llm.transient_retries` defaults to **2** additional attempts per generation;
|
|
||||||
explicit zero disables retries; negative is invalid. PromptKit owns which failures
|
|
||||||
are eligible. Do not add an Audita retry loop.
|
|
||||||
- `llm.repair_attempts` defaults to **2** additional generations for both roles;
|
|
||||||
explicit zero disables repair when deliberately requested. PromptKit supports
|
|
||||||
0–3. Positive defaults are exercised in the prototype. Audita owns this common
|
|
||||||
setting and supplies a complete request output contract with the role's embedded
|
|
||||||
schema, avoiding separate user-facing repair defaults in each prompt.
|
|
||||||
- `llm.operation_timeout` defaults to **15m**, must be positive, and uses a Go
|
|
||||||
duration string. It starts before each logical proposal/assessment call and covers
|
|
||||||
preparation, capacity waiting, transient backoff, generation, and repairs. It is
|
|
||||||
separate from whole-process cancellation. Profile `timeout_seconds` defaults to
|
|
||||||
PromptKit's **600** seconds and is applied afresh to each built-in HTTP attempt.
|
|
||||||
It does not bound the combined retry/repair sequence. A stricter caller/transport timeout can
|
|
||||||
also end execution. The two defaults are selected starting points, not throughput
|
|
||||||
measurements. With the default T=2 and R=2, each logical call dispatches at most
|
|
||||||
**9** attempts before deadline/cancellation; proposal plus assessment may use 18.
|
|
||||||
- Omission retains static defaults or invokes the recorded role/backend derivations.
|
|
||||||
Null is not a general synonym for omission: nonoptional scalars reject it. Optional
|
|
||||||
credentials and the future optional section target accept null. Explicit empty
|
|
||||||
module lists will mean an empty correction pipeline, not “restore defaults.”
|
|
||||||
Repeated modules retain their existing ordered meaning.
|
|
||||||
- The original prototype disabled ConfigKit imports, selected configuration layers
|
|
||||||
(then called configuration profiles), and secret-directory bootstrap. Selected
|
|
||||||
overlays are now verified under the linked overlay decision; the other two
|
|
||||||
features remain disabled. PromptKit execution profiles retain their meaning.
|
|
||||||
|
|
||||||
Configuration validation/inspection must assemble and inspect both selected profiles
|
|
||||||
without provider calls. ConfigKit's effective YAML describes only its loaded state;
|
|
||||||
a separate selected-profile view shows effective model controls/backend selection.
|
|
||||||
Do not imply either snapshot alone represents the full execution configuration.
|
|
||||||
The prototype exercises the two APIs but does not implement production subcommands.
|
|
||||||
Do not serialize raw profile/library objects as a permanent Audita report schema.
|
|
||||||
|
|
||||||
## Migration Inventory
|
|
||||||
|
|
||||||
This inventory covers the current configuration and process flags documented in
|
|
||||||
[configuration](../config.md) and [CLI](../cli.md), including environment-only fields.
|
|
||||||
“Retain” below means retain domain meaning in the new schema, not accept old files.
|
|
||||||
All legacy environment overlays listed afterward are removed.
|
|
||||||
|
|
||||||
| Current file field / setting | Current CLI flag(s) | Target and rationale |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| File selection | `--config` | Retain explicit invocation flag; discovery described above. |
|
|
||||||
| Transcript input | positional path | Invocation only; retain one transcript path. |
|
|
||||||
| Glossary input | `--glossary` | Invocation only; retain required glossary path. |
|
|
||||||
| Corrected output | `--output` | Invocation only; retain stdout fallback. |
|
|
||||||
| Report output | `--report-json` | Invocation only; retain separate report file. |
|
|
||||||
| `pipeline.modules` | `--modules` | Retain sequence/default `[glossary, homophones, glossary, spoken_word, grammar]`; explicit empty clears rather than inherits. |
|
|
||||||
| `output.schema` | `--output-schema` | Retain encoder selection/default `bare-segments`. |
|
|
||||||
| `llm.proposal.base_url`, `llm.validation.base_url` | `--base-url`, `--validation-base-url` | Replace with backend endpoint; remove per-role endpoint flags. |
|
|
||||||
| Both `llm.*.model` | `--model`, `--validation-model` | Replace with profile model; new `--proposal-profile` / `--validation-profile` select whole profiles. |
|
|
||||||
| Both `llm.*.api_key_env`, direct runtime keys | `--llm-api-key`, `--validation-llm-api-key` | Replace with backend ConfigKit secret reference; remove direct key flags. |
|
|
||||||
| Both `llm.*.timeout` | `--llm-timeout-seconds`, `--validation-llm-timeout-seconds` | Replace with profile `timeout_seconds`, plus `llm.operation_timeout`; no duplicate timeout flags. |
|
|
||||||
| Both `llm.*.max_retries` | `--max-retries`, `--validation-max-retries` | Replace with common `llm.transient_retries` and distinct `llm.repair_attempts`; remove misleading old flags. |
|
|
||||||
| `concurrency.total_llm` | `--total-llm-concurrency`, `--llm-concurrency` | Replace with per-backend concurrency; remove global/alias flags. |
|
|
||||||
| `concurrency.proposal_llm`, `concurrency.validation_llm` | `--proposal-llm-concurrency`, `--validation-llm-concurrency` | Remove role budgets; shared backend handles service capacity. |
|
|
||||||
| Validation prompt budget (no old YAML field) | `--validation-max-prompt-tokens` | Add `validation.max_prompt_tokens`, default 2048, positive; retain explicit flag override. Audita owns batching. |
|
|
||||||
| `chunking.target_sections` | `--target-sections` | Retain optional positive target; omission/null means automatic, zero invalid. |
|
|
||||||
| `chunking.max_section_tokens` | `--max-section-tokens` | Retain 8192, positive. |
|
|
||||||
| `chunking.min_section_tokens` | `--min-section-tokens` | Retain 2048, positive, no greater than maximum. |
|
|
||||||
| `normalization.max_segment_gap` | `--normalize-max-segment-gap` | Retain 4.0 numeric seconds, non-negative; drop alternate YAML duration syntax for these domain numeric fields. |
|
|
||||||
| `normalization.ellipsis_gap` | `--normalize-ellipsis-gap` | Retain 3.5 numeric seconds, non-negative. |
|
|
||||||
| `normalization.max_segment_duration` | `--normalize-max-segment-duration` | Retain 60.0 numeric seconds, positive. |
|
|
||||||
| `normalization.max_segment_tokens` | `--normalize-max-segment-tokens` | Retain 2048, positive. |
|
|
||||||
| `thresholds.glossary` | `--glossary-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.homophones` | `--homophones-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.spoken_word` | `--spoken-word-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `thresholds.grammar` | `--grammar-confidence-threshold` | Retain 0.8 in [0,1]. |
|
|
||||||
| `context.description` | `--transcript-description` | Retain trimming and 500-byte limit; empty clears context. |
|
|
||||||
| `diagnostics.work_dir` | `--work-dir` | Retain `/tmp/audita`, nonempty; artifact lifecycle remains Audita-owned. |
|
|
||||||
| `diagnostics.retention` | `--work-dir-retention` | Retain `auto` default and selection; existing `never` follow-up is separate. |
|
|
||||||
| Help/version/config commands | `--help`, `-h` | Retain command purpose; help should parse before loading configuration. Effective-config output intentionally becomes ConfigKit YAML, not legacy JSON. |
|
|
||||||
|
|
||||||
Retained setting flags become explicit ConfigKit overrides only when supplied;
|
|
||||||
CLI parser defaults must not overwrite file values. Normalization flag values stay
|
|
||||||
numeric seconds. Domain values convert into existing domain structs after validation;
|
|
||||||
do not spread ConfigKit or PromptKit types into correction algorithms.
|
|
||||||
|
|
||||||
Complete environment removal map (names remain permissible if explicitly named as
|
|
||||||
secret references; none have implicit configuration meaning):
|
|
||||||
|
|
||||||
| Old environment names | Replacement |
|
|
||||||
| --- | --- |
|
|
||||||
| `AUDITA_CONFIG` | File selection above. |
|
|
||||||
| `AUDITA_MODULES` | `pipeline.modules` / explicit flag. |
|
|
||||||
| `AUDITA_LLM_API_KEY`, `OPENROUTER_API_KEY`, `AUDITA_VALIDATION_LLM_API_KEY` | Explicit backend credential references; no implicit fallback. |
|
|
||||||
| `AUDITA_MODEL`, `AUDITA_VALIDATION_MODEL` | Profile files and role selection. |
|
|
||||||
| `AUDITA_BASE_URL`, `AUDITA_VALIDATION_BASE_URL` | Backend endpoints. |
|
|
||||||
| `AUDITA_LLM_TIMEOUT_SECONDS`, `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | Profile timeout and operation deadline. |
|
|
||||||
| `AUDITA_MAX_RETRIES`, `AUDITA_VALIDATION_MAX_RETRIES` | Common transient/repair settings. |
|
|
||||||
| `AUDITA_TOTAL_LLM_CONCURRENCY`, `AUDITA_LLM_CONCURRENCY`, `AUDITA_PROPOSAL_LLM_CONCURRENCY`, `AUDITA_VALIDATION_LLM_CONCURRENCY` | Backend concurrency; remove alias/role budgets. |
|
|
||||||
| `AUDITA_VALIDATION_MAX_PROMPT_TOKENS` | `validation.max_prompt_tokens` / explicit flag. |
|
|
||||||
| `AUDITA_TARGET_SECTIONS`, `AUDITA_MAX_SECTION_TOKENS`, `AUDITA_MIN_SECTION_TOKENS` | Corresponding chunking fields / flags. |
|
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP`, `AUDITA_NORMALIZE_ELLIPSIS_GAP`, `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION`, `AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS` | Corresponding normalization fields / flags. |
|
|
||||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD`, `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD`, `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD`, `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | Corresponding threshold fields / flags. |
|
|
||||||
| `AUDITA_WORK_DIR`, `AUDITA_WORK_DIR_RETENTION` | Diagnostics fields / flags. |
|
|
||||||
|
|
||||||
## Execution and Failure Mapping
|
|
||||||
|
|
||||||
The executable uses grammar proposal plus grammar review, current response schemas,
|
|
||||||
current hardening text, transcript-section/description helpers, proposal preview, and
|
|
||||||
actual decision-cardinality enforcement from Audita v1.1.0. PromptKit renders the
|
|
||||||
adapted templates; the custom HTTP transport supplies OpenAI-compatible responses.
|
|
||||||
The prototype pre-expands Audita's `hardening` template function into the embedded
|
|
||||||
PromptKit definitions; it passes existing named string variables unchanged. No
|
|
||||||
provider, paid call, or injected high-level LLM client participates.
|
|
||||||
|
|
||||||
Generated JSON is structurally validated/repaired by PromptKit before Audita decodes
|
|
||||||
it. Audita then checks proposal applicability, confidence, and decision identity and
|
|
||||||
cardinality. The demonstration previews corrections without mutation. Production
|
|
||||||
retains every module's full validator chain, glossary protection, batching, and
|
|
||||||
apply-time checks; the representative prototype does not claim to replace them.
|
|
||||||
|
|
||||||
| Outcome | Section/batch effect | Overall run |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Structural repair succeeds | Continue domain validation. | Normal execution. |
|
|
||||||
| Proposal repair exhausted | No proposals from this section; malformed-output warning. | Continue other work; successful completion may include warnings. |
|
|
||||||
| Assessment repair exhausted | Reject this batch; malformed-output warning. | Continue other work. |
|
|
||||||
| Schema-valid invalid/missing/duplicate decision indices | Reject batch and warn; no semantic retry/repair. | Continue other work. |
|
|
||||||
| Inapplicable/low-confidence proposal or negative assessment | Reject affected proposal; preserve domain reasons. | Continue other work. |
|
|
||||||
| Provider failure, retry exhaustion, transport or malformed HTTP response | Stop affected operation; retain available observations. | Runtime failure; cancel remaining work, retain existing Audita diagnostic handling. |
|
|
||||||
| Deadline, caller cancellation, capacity admission failure | Stop affected operation; no outer retry or fabricated output. | Runtime failure with distinct classified cause. |
|
|
||||||
| Invalid settings/profile/schema setup | Fail before processing where detectable. | Configuration error for user input, runtime error for broken embedded assets. |
|
|
||||||
|
|
||||||
Zero transient retries produces one ordinary provider failure, not
|
|
||||||
`ErrRetriesExhausted`; the latter signals an exhausted **positive** budget.
|
|
||||||
Provider failures retain a typed cause. Do not expose raw provider messages through
|
|
||||||
user diagnostics: continue Audita's separate secret-redaction boundary. A failed
|
|
||||||
logical call may return no result while its observations still record completed
|
|
||||||
attempts or an earlier successful generation. Already completed domain results can
|
|
||||||
remain in diagnostics; failure must not be presented as a fully corrected output.
|
|
||||||
|
|
||||||
## Observation and Report Handoff
|
|
||||||
|
|
||||||
The prototype maps library events into Audita-owned records under a short mutex.
|
|
||||||
Records contain logical run/role/backend/profile identity, generation and attempt
|
|
||||||
indices, dispatch state, queue/generation/retry/wall timing, classified outcomes,
|
|
||||||
HTTP status, exhaustion, repair/retry counts, and each usage field's value,
|
|
||||||
knownness, and completeness. It omits URLs, credentials, payloads, raw provider
|
|
||||||
errors, and prompt text. Identifiers themselves are metadata, not a secret channel.
|
|
||||||
|
|
||||||
Use terminal records once for logical totals; retain attempt records for diagnosis
|
|
||||||
without adding them to terminal totals. A known zero is distinct from unknown;
|
|
||||||
partial totals remain incomplete after unmetered failed attempts. These observations
|
|
||||||
are not a billing ledger. Callback code must not wait on its own run's progress.
|
|
||||||
|
|
||||||
Production report changes must distinguish logical calls from dispatched provider
|
|
||||||
attempts. Replace scheduler utilization fields based on global/proposal/validation
|
|
||||||
pools with per-backend admission/active timing. Preserve deterministic ledger and
|
|
||||||
validator ordering. Map prompt definition/rendered hashes and selected profile/model
|
|
||||||
into request metadata, but do not promise equality with legacy prompt hashes or
|
|
||||||
serialize PromptKit's raw event struct as the stable report contract. Update the
|
|
||||||
output integration reference, report mapper tests, and downstream migration guide
|
|
||||||
when the production representation is implemented.
|
|
||||||
|
|
||||||
## Verification and Remaining Handoff
|
|
||||||
|
|
||||||
The focused matrix is implemented in the nested module:
|
|
||||||
|
|
||||||
| Roadmap scenario | Evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| Successful assembled path | `go run .`; `TestWorkflowBoundaries/repair_both_roles`; actual Audita domain helpers and both selected profiles. |
|
|
||||||
| Presence, overrides, inheritance | `TestConfigurationContract`: omitted role/backend values, explicit zero, null/empty, version, unknown keys, and explicit overrides. |
|
|
||||||
| Credentials | Missing/unset and empty references fail; optional null accepted; effective/profile inspection redacted; HTTP authorization verified; no secret/prompt text in observations. `TestProfilesRespectSharedBackendBoundary` rejects competing lookup and capacity bypass. |
|
|
||||||
| Both roles repaired | `repair_both_roles`: four attempts, two repairs, one accepted correction. |
|
|
||||||
| Repair exhausted | Proposal and assessment exhaustion subtests: bounded counts and selected warning/rejection outcomes. |
|
|
||||||
| Semantic boundary | Missing span, missing/duplicate decision indices, negative assessment; no semantic repair loop. |
|
|
||||||
| Retry/zero/exhaustion | 429 recovery, zero-budget 503, positive-budget exhaustion after completed proposal; typed error assertions. |
|
|
||||||
| Combined budgets | 503 → invalid JSON → repair 503 → valid proposal → assessment: five attempts, two transient retries, one repair. |
|
|
||||||
| Shared capacity | `TestSharedCapacityAndRecovery`: caps four and two, both roles, a retry and repair, queue rejection, measured peak equal to cap, later successful call. |
|
|
||||||
| Waiting/cancellation/deadline | `TestQueuedCancellationAndDeadline`; `TestDeadlineStopsRetryAfter`: no queued dispatch after cancellation/deadline and a 60-second provider retry delay interrupted by the operation deadline. |
|
|
||||||
| Failed-run observations | `TestPartialUsageOnFailedRun`: valid HTTP output with known-zero usage followed by repair failure; nil result, retained events, incomplete known aggregate, correct counts. |
|
|
||||||
| Representative assets | `TestRealPromptRenderingParity` compares both rendered message pairs to Audita v1.1.0; `TestSchemaFixturesMatchAudita` compares schema bytes. |
|
|
||||||
|
|
||||||
Commands executed from `examples/library-adoption` on Linux amd64 with Go 1.26.8:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
GOWORK=off go mod download all
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go mod verify
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go run .
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go test -race ./...
|
|
||||||
GOWORK=off GOPROXY=off GOSUMDB=off go vet ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
All passed. The demonstration reports accepted=1, rejected=0, attempts=4,
|
|
||||||
transient_retries=0, structural_repairs=2. Module verification reports all modules
|
|
||||||
verified. Published tag availability was checked with `git ls-remote --tags` for
|
|
||||||
ConfigKit v0.3.1 and PromptKit v0.11.0; successful module acquisition verifies this
|
|
||||||
is not merely a local-checkout experiment. The module has no replacements.
|
|
||||||
|
|
||||||
The minimum Go 1.25.5 toolchain was also exercised with offline tests and the same
|
|
||||||
successful demonstration. The exact installed-toolchain invocation was:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go test ./...
|
|
||||||
GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off /home/eric/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.25.5.linux-amd64/bin/go run .
|
|
||||||
```
|
|
||||||
|
|
||||||
An initial attempt to select that cached toolchain through `GOTOOLCHAIN=go1.25.5`
|
|
||||||
with checksum verification disabled was rejected by Go's toolchain verifier. Using
|
|
||||||
the installed binary with `GOTOOLCHAIN=local` proved offline execution without
|
|
||||||
weakening dependency checks. The portable README commands assume an installed
|
|
||||||
sufficiently new Go executable on PATH and disable toolchain switching.
|
|
||||||
|
|
||||||
From the Audita root, `go test ./...` and
|
|
||||||
`go build -o ./bin/audita ./cmd/audita` passed. These check existing production code;
|
|
||||||
they do not cover the nested module. Affected relative documentation links,
|
|
||||||
inventory coverage, and `git diff --check` were checked separately.
|
|
||||||
|
|
||||||
### Findings and Limits
|
|
||||||
|
|
||||||
No adoption-blocking upstream API gap was found. Two integration details deserve
|
|
||||||
explicit attention: PromptKit's profile GoDoc calls `timeout_seconds` a generation
|
|
||||||
deadline, but its built-in HTTP path applies it separately to each attempt; the
|
|
||||||
consumer therefore needs the selected total-operation deadline. Also,
|
|
||||||
`APIKeyRequired` is an in-memory profile facility, not a profile-YAML field. The
|
|
||||||
file-profile design uses the presence of ConfigKit's optional credential reference
|
|
||||||
to require local secret resolution; it does not invent a new PromptKit YAML key.
|
|
||||||
|
|
||||||
The chosen defaults are proposed operational policy, not measured model-quality or
|
|
||||||
cost recommendations. Tests use controlled transport responses and synthetic data;
|
|
||||||
they do not establish real provider compatibility, model quality, network behavior,
|
|
||||||
throughput, cross-process coordination, complete billing, or release compliance.
|
|
||||||
Only the representative LLM-facing subset is implemented in the prototype; the
|
|
||||||
inventory specifies the remaining domain fields for the ConfigKit migration.
|
|
||||||
Production file discovery, CLI parsing, full module chains, all nine prompts,
|
|
||||||
filesystem diagnostics/redaction, and serialized report migration remain their
|
|
||||||
assigned production tasks. Those are deliberate scope boundaries, not missing
|
|
||||||
claims of completion for the prototype.
|
|
||||||
|
|
||||||
The new profile/credential/capacity and deadline choices are explicit above; no
|
|
||||||
unresolved upstream requirement prevents starting the ConfigKit implementation
|
|
||||||
plan. Production report field names and final runtime wiring are selected in the
|
|
||||||
[implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md), following the outcome and observation
|
|
||||||
contract here. The coordinated refactor targets both libraries directly; intermediate commits do not require a working legacy runtime.
|
|
||||||
|
|
||||||
### Production Implementation Reference
|
|
||||||
|
|
||||||
The [implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) owns ordered production work,
|
|
||||||
component changes, tests, and acceptance criteria. The [combined roadmap](library-adoption.md)
|
|
||||||
and [configuration roadmap](configkit.md) own the final target. This evidence does
|
|
||||||
not require an interim runtime bridge or separately runnable configuration migration.
|
|
||||||
|
|
||||||
Retained domain scope includes transcript/glossary schemas, normalization/chunking,
|
|
||||||
output encoding, proposals and apply-time checks, deterministic validators, ordered
|
|
||||||
modules, LLM-validator batching/cardinality, ledger ordering, and artifact lifecycle.
|
|
||||||
The production integration replaces loading, provider execution, prompt rendering,
|
|
||||||
and scheduler composition while preserving those responsibilities. Retention and
|
|
||||||
timestamp changes remain separate work.
|
|
||||||
|
|
||||||
### Distribution Follow-Up
|
|
||||||
|
|
||||||
The pinned graph includes ConfigKit, PromptKit, both PromptKit backend catalogs,
|
|
||||||
jsonschema/v6, yaml.v3, yaml.v4, x/text, and kr/text (including module/test dependency
|
|
||||||
edges). Before distribution, review the actual production dependency graph and
|
|
||||||
its license files, including PromptKit/catalog LGPL-3.0-or-later terms. Decide and
|
|
||||||
verify the notices, license copies, corresponding library source/modifications,
|
|
||||||
and rebuild/relink materials needed for Audita's Go binary/container distribution.
|
|
||||||
This unit does not publish binaries or claim that release compliance is complete.
|
|
||||||
No upstream relicensing request or library implementation change is required for
|
|
||||||
this prototype.
|
|
||||||
|
|
||||||
## Selected-Overlay Extension: 2026-09-24
|
|
||||||
|
|
||||||
The isolated prototype now pins published ConfigKit v0.4.0. ConfigKit loads
|
|
||||||
`overlays/testing.yml` relative to the selected main file; the overlay selects
|
|
||||||
complete PromptKit execution profiles from the separate `profiles/` directory.
|
|
||||||
The original v0.3.1 observations above are historical and do not prove this
|
|
||||||
extension. No production Audita behavior changed.
|
|
||||||
|
|
||||||
With Go 1.26.8 on linux/amd64, from `examples/library-adoption/`, these offline
|
|
||||||
checks passed with `GOTOOLCHAIN=local GOWORK=off GOPROXY=off GOSUMDB=off`:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go run .
|
|
||||||
go run . --overlay testing
|
|
||||||
go run . --overlay testing --proposal-profile proposal
|
|
||||||
```
|
|
||||||
|
|
||||||
Each demonstration reported one accepted correction, four attempts, zero
|
|
||||||
transient retries, and two structural repairs. The base run selected
|
|
||||||
`synthetic-grammar`/`synthetic-review`; testing selected
|
|
||||||
`synthetic-grammar-testing`/`synthetic-review-testing`; the explicit proposal
|
|
||||||
override restored only `synthetic-grammar`. These are synthetic routing IDs,
|
|
||||||
not price, latency, quality, or provider-compatibility measurements.
|
|
||||||
|
|
||||||
| Requirement | Consumer evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| Base, empty/default, testing selection, role overrides, final inheritance, and relative profile resolution | `TestSelectedOverlayConfiguration` loads temporary on-disk layouts and assembles both roles. |
|
|
||||||
| Actual base/testing/override HTTP models and repairs | `TestDemonstrationSelectsOutboundModels` captures every request body and matches model IDs to the reported selections and domain outcome. |
|
|
||||||
| Missing/invalid selection, version/directive rejection, and ordinary backend-setting validation | `TestOverlayFailuresAndApplicationValidation` checks ConfigKit diagnostic codes and effective backend limits. |
|
|
||||||
| Surviving credential resolution, absent/empty failures, null unauthenticated use, and profile-only switching | `TestOverlayCredentialResolutionAndInspection` controls synthetic environment values and observes HTTP Authorization. |
|
|
||||||
| Redacted effective output, explanation and metadata origin, profile inspection, and observations | `TestOverlayCredentialResolutionAndInspection` and `TestOverlaySelectionProvenance` inspect safe output and overlay/CLI origins. |
|
|
||||||
| Existing retry, repair, domain, capacity, and concurrent-observation behavior | `TestWorkflowBoundaries`, `TestSharedCapacityAndRecovery`, `TestQueuedCancellationAndDeadline`, `TestDeadlineStopsRetryAfter`, and `TestPartialUsageOnFailedRun` pass, including under `-race`. |
|
|
||||||
|
|
||||||
Changing a profile does not remove a backend credential reference from the
|
|
||||||
effective configuration. ConfigKit validates a surviving reference when only
|
|
||||||
profile selection changes. Replacing the reference in an
|
|
||||||
overlay resolves only the replacement; explicit null allows unauthenticated use.
|
|
||||||
This distinction is part of the selected configuration boundary, not an
|
|
||||||
application fallback. Imports and secret-directory bootstrap remain disabled.
|
|
||||||
|
|
||||||
The tests use a controlled built-in HTTP transport and synthetic credentials;
|
|
||||||
there were no live provider calls. The prototype still covers one representative
|
|
||||||
proposal/assessment pair, not all production modules, prompts, CLI paths, or
|
|
||||||
diagnostic artifacts. Root `go test ./...` does not run this nested module. No
|
|
||||||
upstream API gap appeared in this extension. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
|
||||||
keeps implementation and release pending.
|
|
||||||
@@ -1,269 +0,0 @@
|
|||||||
# Library Adoption: Shared Boundary and Prototype
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
**Status:** Original boundary-design and offline-prototype unit completed;
|
|
||||||
overlay extension and production-planning handoff completed, 2026-09-24.
|
|
||||||
The work packages below retain the original scope;
|
|
||||||
[decisions, evidence, limitations, and handoff](library-boundary-findings.md)
|
|
||||||
record the outcome. The executable prototype was removed after production
|
|
||||||
integration; Git history at `590604b` retains it. No publication or upstream
|
|
||||||
changes were made in that historical unit.
|
|
||||||
|
|
||||||
## Next Unit: Testing Overlays and Production Migration Plan
|
|
||||||
|
|
||||||
**Status:** Complete. The [production implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md)
|
|
||||||
owns subsequent work; the [overlay findings](library-boundary-findings.md#selected-overlay-extension-2026-09-24)
|
|
||||||
record this unit's checks. The original work packages below remain
|
|
||||||
historical scope for the completed first unit.
|
|
||||||
|
|
||||||
Extend the existing prototype to prove the [selected-overlay contract](configkit.md#selected-overlays-for-testing)
|
|
||||||
with ConfigKit v0.4.0, then turn the verified findings into a detailed production
|
|
||||||
ConfigKit migration plan. Do not repeat the original inventory or redesign the
|
|
||||||
established library boundary without new evidence.
|
|
||||||
|
|
||||||
Required work:
|
|
||||||
|
|
||||||
- Upgrade only the nested prototype module from ConfigKit v0.3.1 to v0.4.0.
|
|
||||||
- Pass an explicit overlay selection through ConfigKit and add `overlays/testing.yml`
|
|
||||||
plus two PromptKit testing profiles. Select both roles and retain one shared
|
|
||||||
finite backend. Synthetic model names demonstrate routing; they do not benchmark
|
|
||||||
real model cost, speed, or quality.
|
|
||||||
- Make the offline executable accept overlay and explicit role-profile selections,
|
|
||||||
and show the selected models alongside domain outcomes and retry/repair counts.
|
|
||||||
- Verify base/overlay/CLI precedence, role inheritance, invalid selections,
|
|
||||||
surviving credential references, safe inspection, and overlay provenance.
|
|
||||||
Prove the actual HTTP request models for both roles; loading changed strings
|
|
||||||
alone is insufficient evidence.
|
|
||||||
- Record new results separately from the historical v0.3.1 findings. Expand the
|
|
||||||
existing ConfigKit roadmap into a concrete production implementation handoff,
|
|
||||||
including final runtime integration and removal of legacy configuration.
|
|
||||||
|
|
||||||
Target end state: reviewers can run the prototype with and without the testing
|
|
||||||
selection and see both LLM roles switch models. The offline test suite establishes
|
|
||||||
that selection, inspection, and execution agree, and the production ConfigKit work
|
|
||||||
has bounded tasks with explicit acceptance criteria.
|
|
||||||
|
|
||||||
Acceptance criteria:
|
|
||||||
|
|
||||||
- The published v0.4.0 dependency is pinned without local replacements; the original
|
|
||||||
no-overlay path and existing integration tests continue to pass.
|
|
||||||
- `testing` switches both roles, explicit profile flags override its selections,
|
|
||||||
and omission/empty/`default` retains base selection according to ConfigKit.
|
|
||||||
- Invalid overlays fail clearly; inheritance and secret resolution use surviving
|
|
||||||
effective settings. Inspection records overlay origin without disclosing keys.
|
|
||||||
- Both selected testing models reach the controlled HTTP transport; structural
|
|
||||||
repair, shared capacity, observations, and Audita domain checks remain active.
|
|
||||||
- Updated examples, executable instructions, findings, and a detailed production
|
|
||||||
migration plan are reviewable. Exact verification commands and limitations are
|
|
||||||
recorded, including the fact that root tests do not cover the nested module.
|
|
||||||
- Production code, root dependencies, and upstream libraries remain unchanged.
|
|
||||||
No release or live provider call is part of this unit.
|
|
||||||
|
|
||||||
This unit ends with a production migration **plan**, not production implementation.
|
|
||||||
The ConfigKit → PromptKit → combined release sequence remains unchanged. Imports
|
|
||||||
and secret-directory bootstrap remain disabled; PromptKit execution profiles keep
|
|
||||||
their existing names and meaning.
|
|
||||||
|
|
||||||
## Objective
|
|
||||||
|
|
||||||
Define and prove the boundary between Audita, ConfigKit, and PromptKit before
|
|
||||||
replacing Audita's production configuration or LLM runtime. This unit covers the
|
|
||||||
shared design and representative prototype supporting the
|
|
||||||
[coordinated adoption roadmap](library-adoption.md). The
|
|
||||||
[ConfigKit roadmap](configkit.md) continues to own the broader configuration scope.
|
|
||||||
|
|
||||||
The deliverable is a concrete target contract supported by an offline executable
|
|
||||||
prototype, sufficient to implement the ConfigKit migration without subsequently
|
|
||||||
redesigning its LLM-facing settings. A successful prototype is not completion of
|
|
||||||
either production migration.
|
|
||||||
|
|
||||||
## Established Direction
|
|
||||||
|
|
||||||
- This is an intentional breaking migration with a migration guide, not legacy
|
|
||||||
configuration support, aliases, or a permanent compatibility mode.
|
|
||||||
- ConfigKit owns shared configuration mechanics; PromptKit owns shared prompt and
|
|
||||||
generation mechanics; Audita retains domain meaning and transcript correction.
|
|
||||||
- Structural repair is enabled from the outset for proposal and LLM-validator calls.
|
|
||||||
Positive default repair budgets must be selected, not silently left at zero.
|
|
||||||
- Transient retries and structural repair are separate. Explicit zero disables
|
|
||||||
transient retries; Audita must not add a second automatic retry loop.
|
|
||||||
- One finite concurrency limit per backend protects local capacity and bounds paid
|
|
||||||
API activity. Calls from both roles share that limit. Fine-grained role budgets
|
|
||||||
and an additional global concurrency control are unnecessary.
|
|
||||||
- Keep the sequence: boundary/prototype, ConfigKit migration, PromptKit migration,
|
|
||||||
then combined major release. No intermediate public release is planned.
|
|
||||||
|
|
||||||
## Scope and Deliverables
|
|
||||||
|
|
||||||
| Deliverable | Required result |
|
|
||||||
| --- | --- |
|
|
||||||
| Configuration inventory and migration map | Every existing setting and invocation option has a proposed destination, replacement, or removal, with rationale. |
|
|
||||||
| Boundary specification | One authoritative source for each setting, explicit precedence/defaults, credential flow, runtime conversion, and failure handling. |
|
|
||||||
| Representative executable prototype | New-style configuration drives one proposal and one assessment through released libraries and a controlled provider. |
|
|
||||||
| Focused behavioral evidence | Retry, repair, capacity, credential, observation, and domain-boundary scenarios below are demonstrated. |
|
|
||||||
| Decision and integration handoff | Concrete choices, measured limitations, affected components, removal plan, and next implementation tasks are recorded. |
|
|
||||||
|
|
||||||
Do not switch the production CLI to the prototype, migrate every module/prompt,
|
|
||||||
redesign transcript/glossary formats, change correction algorithms, modify retention
|
|
||||||
or timestamp behavior, or publish a release in this unit. Do not reproduce the
|
|
||||||
libraries' full test suites. Upstream changes require a separately scoped task if
|
|
||||||
an actual gap is found.
|
|
||||||
|
|
||||||
## Work Package 1: Inventory and Decide Ownership
|
|
||||||
|
|
||||||
Read the current [configuration](../config.md), [CLI](../cli.md),
|
|
||||||
[LLM runtime](../internal/llm-runtime.md), [prompts](../internal/prompts.md),
|
|
||||||
[validators](../internal/validators.md), and [output contracts](../integrations/output-files.md).
|
|
||||||
Use graph-first discovery to verify the relevant code and tests, especially
|
|
||||||
configuration application, prompt construction, structured responses, scheduling,
|
|
||||||
and report assembly.
|
|
||||||
|
|
||||||
Build a migration table grouped by application/domain settings, invocation state,
|
|
||||||
backend settings, execution profiles, output contracts, and inspection. Include
|
|
||||||
current environment settings and aliases without treating their existence as a
|
|
||||||
requirement to preserve them. Identify report fields whose meanings change when
|
|
||||||
one logical generation contains multiple provider attempts.
|
|
||||||
|
|
||||||
Resolve these design decisions explicitly:
|
|
||||||
|
|
||||||
| Decision | Proposed direction to prove |
|
|
||||||
| --- | --- |
|
|
||||||
| File selection and version | Audita selects the main configuration and validates a new schema version; specify discovery and the treatment of old versions. |
|
|
||||||
| Application versus invocation state | ConfigKit loads persistent settings; Audita parses invocation-only input/output paths and translates explicitly supplied setting flags into supported overrides. |
|
|
||||||
| Backend identity | One registration represents the service capacity shared by proposal and validation calls. Avoid separate identities for the same service merely because roles differ. |
|
|
||||||
| Endpoint and capacity ownership | Define them once per backend, with finite limits and an explicit policy for waiting capacity/exhaustion. Do not expose endpoint-only unlimited execution accidentally. |
|
|
||||||
| Execution profiles | Prefer complete application-owned profiles using PromptKit conventions. Decide their source and selection without duplicating model/settings fields in application configuration. Catalog selection remains a deliberate option. |
|
|
||||||
| Role selection and inheritance | Specify how proposal and validation choose profiles, including omission and any inheritance. Do not inherit old behavior solely for compatibility. |
|
|
||||||
| Credentials | Prefer one ConfigKit-owned secret-resolution path passed through PromptKit's supported request credential field. Define backend/profile credential association and avoid competing environment lookups. Validate this against profile requirements. |
|
|
||||||
| Retry, repair, and deadlines | Assign one owner and explicit defaults for each control. Use positive structural-repair defaults and document combined limits and total-operation scope. |
|
|
||||||
| Optional library features | The original prototype omitted imports, selected configuration layers, and secret-directory bootstrap. Selected overlays are now planned for testing; see the [overlay plan](configkit.md#selected-overlays-for-testing). Imports and secret-directory bootstrap remain out. |
|
|
||||||
| Inspection | Define effective configuration and selected-profile inspection without implying that one library's snapshot contains the other's complete state. |
|
|
||||||
|
|
||||||
Choose concrete field names, example values, defaults, units, and omission/null/zero
|
|
||||||
semantics for the prototype. In particular, decide what validation and inspection
|
|
||||||
do when configured credentials are unavailable. Classify choices as proposed,
|
|
||||||
selected, or unresolved; do not present candidates as implemented contracts.
|
|
||||||
|
|
||||||
## Work Package 2: Specify Execution and Reporting Semantics
|
|
||||||
|
|
||||||
Document the flow from selected configuration through backend/profile assembly,
|
|
||||||
prompt inputs, generation, structural validation/repair, typed decoding, and Audita
|
|
||||||
domain validation. Identify which values cross the boundary and which component
|
|
||||||
owns each conversion. Keep library types out of unrelated domain code where a
|
|
||||||
small existing domain value is sufficient.
|
|
||||||
|
|
||||||
Use the current correction-set and validator-decision schemas as representative
|
|
||||||
inputs. Reuse domain checks where practical. Select one real proposal/validator
|
|
||||||
pair; account for prompt helpers, template variables, shared fragments, and schema
|
|
||||||
references rather than testing only an unrelated hello-world prompt.
|
|
||||||
|
|
||||||
Record the following distinctions:
|
|
||||||
|
|
||||||
- Repair seeks output conforming to its structural contract. Success does not
|
|
||||||
establish semantic correctness or authorize applying a correction.
|
|
||||||
- A schema-valid decision set still requires Audita's cardinality and proposal
|
|
||||||
identity checks. Semantic rejection is not automatically another repair request.
|
|
||||||
- Exhausted structural repair yields failed content validation; provider failures,
|
|
||||||
retry exhaustion, cancellation, and capacity rejection have separate outcomes.
|
|
||||||
- Specify each outcome's intended effect on an Audita section/batch and overall run,
|
|
||||||
including warnings, rejected work, partial domain results, and terminal failures.
|
|
||||||
Preserve domain intent without preserving obsolete transport behavior by accident.
|
|
||||||
- For transient retry budget T and structural repair budget R, one logical call can
|
|
||||||
dispatch up to `(T + 1) * (R + 1)` provider attempts. For example, T=2 and R=2
|
|
||||||
permits nine attempts; these illustrative values do not select production defaults.
|
|
||||||
- Choose the total-operation deadline scope for a logical proposal or assessment;
|
|
||||||
distinguish it from per-attempt timeouts and whole-process cancellation. Provider
|
|
||||||
retry timing may exceed local backoff, so deadline behavior must be explicit.
|
|
||||||
|
|
||||||
Design the Audita observation record and aggregation rules. Include logical-call
|
|
||||||
and attempt identity, role/backend correlation, queue/generation/retry timing,
|
|
||||||
repair and retry counts, outcomes, and available usage. Preserve unknown/incomplete
|
|
||||||
usage rather than inventing zeros or complete costs. Do not add terminal totals to
|
|
||||||
attempt totals. Collect events safely across concurrent calls, without blocking
|
|
||||||
callbacks on progress by the same run or persisting raw library events as a stable
|
|
||||||
Audita schema. Transcript-bearing artifacts remain separately governed by Audita.
|
|
||||||
|
|
||||||
## Work Package 3: Build the Bounded Offline Prototype
|
|
||||||
|
|
||||||
Use ConfigKit v0.3.1 and PromptKit v0.11.0 as reviewed starting points; verify their
|
|
||||||
published availability and APIs before pinning the prototype. Recheck the combined
|
|
||||||
Go minimum (the reviewed PromptKit release requires Go 1.25.5). Do not depend on
|
|
||||||
sibling checkouts, an active Go workspace, or committed local module replacements.
|
|
||||||
|
|
||||||
A proposed home is `examples/library-adoption/` as an explicitly experimental,
|
|
||||||
standalone Go module, isolating prototype dependencies from the production binary.
|
|
||||||
Confirm this placement during execution. Its README must give exact offline run
|
|
||||||
and test commands after dependency acquisition; root `go test ./...` must not be
|
|
||||||
claimed to cover a nested module. Keep complete proposed configuration and prompt
|
|
||||||
fixtures with it and clearly label them as prototype-only.
|
|
||||||
|
|
||||||
The prototype should:
|
|
||||||
|
|
||||||
1. Load representative application settings with ConfigKit, using synthetic secret
|
|
||||||
values supplied by the test environment and an application-owned version rule.
|
|
||||||
2. Assemble shared backend registrations and proposal/validation profile selections.
|
|
||||||
3. Supply representative transcript/proposal inputs to PromptKit definitions and
|
|
||||||
actual response schemas using supported sources.
|
|
||||||
4. Use the built-in HTTP generation path with a controlled local endpoint or
|
|
||||||
transport. An injected LLM client alone cannot prove built-in HTTP retry behavior.
|
|
||||||
5. Run proposal generation, decode and check candidates, run assessment, and perform
|
|
||||||
the selected Audita decision checks without modifying real transcript files.
|
|
||||||
6. Collect selected safe observations and show a compact result understandable to a
|
|
||||||
reviewer, including repaired output and final domain acceptance/rejection.
|
|
||||||
|
|
||||||
No paid calls, real credentials, host environment assumptions, persistent caches,
|
|
||||||
or general-purpose test framework are needed. Make concurrency and cancellation
|
|
||||||
checks deterministic through controlled synchronization. Keep delays bounded and
|
|
||||||
use library-supported controls; do not patch library internals to make tests pass.
|
|
||||||
|
|
||||||
## Focused Verification Matrix
|
|
||||||
|
|
||||||
| Scenario | Evidence required |
|
|
||||||
| --- | --- |
|
|
||||||
| Successful assembled path | ConfigKit-loaded selections drive both real prompt roles through PromptKit and Audita checks. |
|
|
||||||
| Presence, overrides, and inheritance | Representative omission, explicit values, and role selection follow the proposed contract; unknown fields and unsupported versions fail clearly. |
|
|
||||||
| Credentials | Required missing/empty references fail as intended; controlled calls receive the chosen synthetic credential; inspection and observations do not expose it. |
|
|
||||||
| Structural repair succeeds | Invalid proposal output and invalid assessment output are repaired in their respective workflows; positive budgets are actually exercised. |
|
|
||||||
| Repair exhausted | Repeated invalid output produces the selected Audita outcome and accurate accounting, without an unbounded outer retry. |
|
|
||||||
| Domain rejection | Structurally valid but unacceptable correction or decision is rejected by Audita, demonstrating the semantic boundary. |
|
|
||||||
| Transient retry and explicit zero | An eligible temporary status recovers; zero makes one attempt. Exhaustion retains a useful classified cause. |
|
|
||||||
| Combined repair and retry | A repaired generation encountering a transient failure respects both budgets and reports distinct counts. |
|
|
||||||
| Shared capacity | Concurrent calls from both roles, including retry/repair activity, stay within one selected backend cap; test a representative local cap of four and a finite API cap. |
|
|
||||||
| Waiting, cancellation, and deadline | Cancellation/deadline stops pending work and retry waits; capacity exhaustion has the intended application outcome and later calls can still proceed. |
|
|
||||||
| Failed-run observations | Completed work remains observable after later failure; unknown usage, known zero, and aggregate counts are mapped correctly. |
|
|
||||||
|
|
||||||
Tests should protect Audita's integration decisions and important assembled behavior.
|
|
||||||
Rely on PromptKit's tests for exhaustive HTTP-status, backoff, and scheduler details;
|
|
||||||
rely on ConfigKit's tests for its parser and presence machinery. Use race testing for
|
|
||||||
new concurrent observation collection. Keep consumer checks offline and credential-free.
|
|
||||||
|
|
||||||
## Completion Criteria and Handoff
|
|
||||||
|
|
||||||
This unit is complete only when:
|
|
||||||
|
|
||||||
- The settings/ownership table and migration inventory are concrete enough to start
|
|
||||||
the ConfigKit migration, including file/profile sources, CLI scope, credentials,
|
|
||||||
defaults, finite backend limits, positive repair budgets, and deadline policy.
|
|
||||||
- The offline prototype and focused matrix demonstrate the selected boundary with
|
|
||||||
both libraries together, including both LLM roles and Audita domain checks.
|
|
||||||
- Exact verification commands, results, limitations, and unresolved upstream gaps
|
|
||||||
are recorded. Missing evidence is not reported as successful integration.
|
|
||||||
- No adoption-critical decision remains hidden in prototype code. If a gap prevents
|
|
||||||
settling the boundary, record it and revise the plan before declaring readiness.
|
|
||||||
- The existing Audita production behavior remains intact in this unit. Temporary
|
|
||||||
code is clearly isolated and has a removal/integration destination during the
|
|
||||||
production migrations; no second supported runtime is introduced.
|
|
||||||
- A component-level follow-up identifies code to replace, domain code to retain,
|
|
||||||
tests to update, documentation/migration-guide work, and direct integration of
|
|
||||||
the final configuration and PromptKit runtime.
|
|
||||||
- Licensing/distribution follow-up for the selected dependency graph is identified
|
|
||||||
for the later release stage; this prototype does not claim release compliance.
|
|
||||||
- Roadmap links and examples resolve, appropriate checks pass, and commits contain
|
|
||||||
only this unit's work. Significant settled architectural choices may be recorded
|
|
||||||
in an ADR under the [documentation policy](../policy/documentation.md), while
|
|
||||||
implementation status stays in the roadmaps.
|
|
||||||
|
|
||||||
Retain the overall adoption order unless evidence or the other planned large changes
|
|
||||||
justifies revising it. Completing this unit enables detailed production migration;
|
|
||||||
it does not authorize an intermediate release or require unrelated runtime fixes.
|
|
||||||
@@ -1,252 +0,0 @@
|
|||||||
# Standard Policy Alignment
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-28
|
|
||||||
|
|
||||||
**Status:** Implemented. This roadmap replaces its earlier licensing-only scope
|
|
||||||
and records the accepted end state. [implementation.md](implementation.md)
|
|
||||||
records the ordered work and verification; the adopted policies are in
|
|
||||||
`docs/policy/`.
|
|
||||||
|
|
||||||
## Purpose and Review Revision
|
|
||||||
|
|
||||||
Align Audita with [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy),
|
|
||||||
using identical policy language except where a substantive, locally significant
|
|
||||||
requirement or decision justifies a variation. Adopt upstream wording verbatim
|
|
||||||
for non-substantive differences. Perform the required template customizations
|
|
||||||
without inventing application contracts, restrictions, or guarantees.
|
|
||||||
|
|
||||||
The accepted review compared Audita's current documents directly against upstream
|
|
||||||
commit **`d84a38ebab26f048cafbe59a2246700c35d29843`**, beginning with the
|
|
||||||
[adoption and update guide at that revision](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/adoption.md).
|
|
||||||
Use that revision for this adoption. Historical adoption baselines need not be
|
|
||||||
reconstructed or assessed. A later upstream revision requires deliberate review,
|
|
||||||
not an automatic substitution during implementation.
|
|
||||||
|
|
||||||
The public upstream repository, discovery links, and copied-policy attribution
|
|
||||||
are part of the accepted direction. This supersedes the earlier instruction to
|
|
||||||
omit references to the policy repository.
|
|
||||||
|
|
||||||
## Current Baseline
|
|
||||||
|
|
||||||
- Audita has five adopted policies: agents 1.0, architecture 1.0, documentation
|
|
||||||
1.1, testing 1.0, and release 1.0. None has per-document `Upstream Source`
|
|
||||||
metadata. `meta.md` and `oss.md` are absent.
|
|
||||||
- [AGENTS.md](../../AGENTS.md) routes readers to the contributor guide and agent
|
|
||||||
policy. [docs/development.md](../development.md) has component-specific reading
|
|
||||||
guidance, but lacks the upstream selective Policy Reading Guide and currently
|
|
||||||
directs all tasks to the architecture policy.
|
|
||||||
- [COPYRIGHT](../../COPYRIGHT) and [LICENSE](../../LICENSE) already establish
|
|
||||||
Audita's GPL-3.0-or-later application grant and full GPLv3 text. The grant does
|
|
||||||
not yet identify the scope of separately licensed copied policy documents.
|
|
||||||
- There is no canonical `docs/oss/attribution.md` or local dependency license-text
|
|
||||||
collection. Relevant component notices still require an evidence-based inventory.
|
|
||||||
- The [README](../../README.md) lacks the prescribed final licensing statements.
|
|
||||||
The [release procedure](../release.md) lacks licensing-footer instructions, and
|
|
||||||
the [release workflow](../../.woodpecker/release.yml) does not explicitly reuse
|
|
||||||
checked-in release notes as hosted release bodies.
|
|
||||||
- Audita already releases through one publication command with an atomic source
|
|
||||||
and tag push. Linux amd64 and arm64 are required; macOS arm64 is best-effort
|
|
||||||
development support. Successful publication completes the release without
|
|
||||||
waiting for CI or artifact delivery.
|
|
||||||
|
|
||||||
## Targeted Policy Set and Justified Customizations
|
|
||||||
|
|
||||||
All seven policy files belong under `docs/policy/`. The versions below are those
|
|
||||||
at the selected upstream revision. No existing policy is wholly unchanged;
|
|
||||||
existing architectural decisions and platform commitments need no substantive
|
|
||||||
revision merely to align the text.
|
|
||||||
|
|
||||||
| Policy | Target version | Required alignment and justified local treatment |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `agents.md` | 1.3 | Adopt upstream substantive language verbatim. Retain Audita's graph-first Codebase Discovery section because it is an explicitly selected repository workflow. |
|
|
||||||
| `architecture.md` | 1.3 | Adopt general ownership, boundary, and constraint guidance, then perform the template's required application-specific customization. Preserve Audita's ordered pipeline, ConfigKit/PromptKit responsibilities, deterministic processing, recovery, concurrency, and secret-handling decisions. These describe real project requirements rather than stylistic preferences. |
|
|
||||||
| `documentation.md` | 1.11 | Adopt upstream prose and applicable ownership rows. Retain concrete output-contract ownership, contributor change recipes, and the decision to place operating/recovery guidance in operations instead of restoring troubleshooting documentation. Omit HTTP API and public Go API ownership rows because Audita has neither responsibility. |
|
|
||||||
| `testing.md` | 1.3 | Adopt substantive text verbatim. Its distinction between established contracts and invented test requirements needs no local exception. |
|
|
||||||
| `release.md` | 1.4 | Adopt upstream substantive language. Preserve Linux amd64/arm64 requirements, best-effort macOS arm64, unsupported Windows/macOS amd64, and Audita's explicit decision against human release gates. These are selected support and workflow commitments. |
|
|
||||||
| `meta.md` | 1.0 | Adopt provenance, local authority, and public discovery guidance with accurate metadata and local attribution navigation. No additional substantive exception is needed. |
|
|
||||||
| `oss.md` | 1.8 | Adopt the House Format and maintenance rules, with required project-specific facts and reference links. Preserve Audita's GPL-3.0-or-later application grant, Gitea host, and existing distribution channels. |
|
|
||||||
|
|
||||||
Remove each `Base Policy Template` section after carrying out its instructions.
|
|
||||||
Keep local customizations identifiable and explain their project-specific reason
|
|
||||||
in the appropriate policy; do not maintain alternate paraphrases of shared rules.
|
|
||||||
Architecture customization does not authorize a package redesign or additional
|
|
||||||
threat-model guarantees. Supporting documents are warranted by existing
|
|
||||||
responsibilities and substantive content, not by unused template examples.
|
|
||||||
|
|
||||||
Successful-tag-push completion, asynchronous artifacts, minimal automated gates,
|
|
||||||
and proportionate validation are now shared upstream rules. Replace Audita's
|
|
||||||
parallel explanations with the upstream language rather than describing these
|
|
||||||
as continuing exceptions. Keep concrete commands, remotes, build targets, and
|
|
||||||
publication mechanics in the release procedure with appropriate policy links.
|
|
||||||
|
|
||||||
## Provenance and Selective Reading
|
|
||||||
|
|
||||||
For every adopted policy, retain the selected upstream `Base Policy Version`,
|
|
||||||
set `Last Updated` to its actual local adoption date, and introduce `Upstream
|
|
||||||
Source` linking to that exact file at the selected full commit. The URL pattern is:
|
|
||||||
|
|
||||||
```text
|
|
||||||
https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/<filename>.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Substitute each of `agents`, `architecture`, `documentation`, `testing`, `release`,
|
|
||||||
`meta`, and `oss`; a repository-root or shared generic source link is insufficient.
|
|
||||||
These links describe the source actually incorporated, not a reconstructed claim
|
|
||||||
about older local documents. Git and per-document metadata are sufficient; no
|
|
||||||
separate synchronization registry or recurring check is required.
|
|
||||||
|
|
||||||
In `meta.md`, retain these public discovery destinations:
|
|
||||||
|
|
||||||
- [Standard Repository Policies](https://gitea.maximumdirect.net/eric/policy).
|
|
||||||
- [Current adoption and update guide](https://gitea.maximumdirect.net/eric/policy/src/branch/main/docs/adoption.md).
|
|
||||||
|
|
||||||
Discovery links follow the current upstream project and guide. Adopted-source,
|
|
||||||
notice, and example links remain commit-pinned. Preserve valid pinned example
|
|
||||||
references supplied by the selected policy, or pin them to this review revision
|
|
||||||
after checking the referenced assets; do not turn them into moving-branch links.
|
|
||||||
|
|
||||||
Adapt upstream `agents/AGENTS.md` into Audita's root orientation, retaining its
|
|
||||||
existing applicable instructions. Add the adoption guide's Policy Reading Guide
|
|
||||||
to `docs/development.md` and retain the component-specific task guide. Route
|
|
||||||
policy adoption and updates to `meta.md`, repository recommendations and changes
|
|
||||||
to agents, application-code or structure work to architecture, validation choices
|
|
||||||
to testing, documentation work to documentation, dependency/licensing work to
|
|
||||||
OSS, and release/versioning work to release policy. Read applicable policies
|
|
||||||
before their decisions, reuse unchanged context, and expand reading with scope.
|
|
||||||
Do not require reading the entire suite for every task.
|
|
||||||
|
|
||||||
## Copied Policy Licensing and Local Material
|
|
||||||
|
|
||||||
Follow the selected revision's
|
|
||||||
[canonical policy attribution entry](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/policy-attribution.md)
|
|
||||||
and [project grant](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT).
|
|
||||||
Insert the entry beneath the title in `docs/oss/attribution.md`, substitute the
|
|
||||||
full review commit in all three URLs, and accurately identify copied/adapted
|
|
||||||
material, including orientation or reading-guide text where copied. Preserve
|
|
||||||
supplied creator, copyright, license, disclaimer notices, and modification
|
|
||||||
indications. Keep the material statement concise.
|
|
||||||
|
|
||||||
Copy `CC-BY-SA-4.0.txt` into `docs/oss/licenses/`. Add the adoption guide's scoped
|
|
||||||
notice to `COPYRIGHT`, identifying the adapted policies as CC-BY-SA-4.0 and
|
|
||||||
routing readers to attribution for the affected material. Account accurately for
|
|
||||||
other copied guidance in that scope. Audita's application code remains
|
|
||||||
GPL-3.0-or-later; unrelated documentation is not automatically relicensed.
|
|
||||||
The root GPLv3 `LICENSE` needs no change.
|
|
||||||
|
|
||||||
| Material | Destination or treatment |
|
|
||||||
| --- | --- |
|
|
||||||
| Applicable policies, including `meta.md` and `oss.md` | Copy and customize under `docs/policy/`, retaining provenance. |
|
|
||||||
| Upstream `agents/AGENTS.md` | Adapt into root `AGENTS.md`. |
|
|
||||||
| Selective reading guidance | Integrate into Audita's own `docs/development.md`. |
|
|
||||||
| Upstream README, root AGENTS.md, developer guide, and adoption guide | Keep upstream and consult through links; do not copy as local guides. |
|
|
||||||
| Licensing example collection | Keep upstream with commit-pinned references; use it to create actual notices. Original example formatting is CC0 and alone needs no policy attribution entry. |
|
|
||||||
| Applicable standard and companion license texts | Copy complete, unmodified texts locally; no blanket collection. |
|
|
||||||
|
|
||||||
## Component Attribution
|
|
||||||
|
|
||||||
Complete the previously agreed inventory of relevant incorporated or distributed
|
|
||||||
components: direct and transitive dependencies, copied code/documentation/assets,
|
|
||||||
and applicable Go runtime notices. Determine inclusion from actual material and
|
|
||||||
use, not simply presence in `go.mod`; tooling or consultation alone does not
|
|
||||||
require an entry. Inspect selected sources directly rather than copying a
|
|
||||||
dependency's attribution document wholesale.
|
|
||||||
|
|
||||||
Use the exact House Format in OSS policy 1.8: the prescribed title followed by
|
|
||||||
component entries, factual identification, applicable supplied copyrights and
|
|
||||||
SPDX expressions, standard license/notice labels, and `[Source](COMMIT_URL).`
|
|
||||||
links. Omit unavailable optional fields instead of adding explanatory filler.
|
|
||||||
Preserve required upstream material, license alternatives, and exceptions.
|
|
||||||
Keep research narratives, compliance conclusions, and exception reasoning out
|
|
||||||
of attribution; place reasoning in the OSS policy or an appropriate decision
|
|
||||||
record. This replaces the older roadmap's permission to put exception reasoning
|
|
||||||
in a component entry.
|
|
||||||
|
|
||||||
Upstream component and notice links identify exact imported commits. Local
|
|
||||||
standard texts include directly reachable companions, such as GPLv3 for LGPLv3.
|
|
||||||
Apply the adopted house interpretation of notice links without blanket extra
|
|
||||||
local-copy requirements or repeated approval. Concrete obligations beyond notice
|
|
||||||
reproduction receive their own applicable treatment; the format alone does not
|
|
||||||
certify compatibility or resolve every distribution obligation.
|
|
||||||
|
|
||||||
The reviewed dependency revisions remain useful starting evidence, subject to
|
|
||||||
checking the dependencies selected when implementation occurs:
|
|
||||||
|
|
||||||
| Component | Reviewed version | Source commit |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| ConfigKit | v0.4.1 | `0adb94f59e44827908eff401d4381a398fe17107` |
|
|
||||||
| PromptKit | v0.11.1 | `15e40679546f2e21b6f51cc84b6f0a087a8486db` |
|
|
||||||
| OpenRouter catalog | v2.20260926.0 | `4f731d217e0bc39151c010f50fb5912706e7b6b3` |
|
|
||||||
| Rakestrawhome catalog | v2.20260926.0 | `48918a3ad39fd99843cda94a86133b1b3881d596` |
|
|
||||||
|
|
||||||
These library patches changed licensing/documentation rather than library code
|
|
||||||
or catalog profiles. ConfigKit and PromptKit identify LGPL-3.0-or-later; the
|
|
||||||
selected catalogs identify MIT. Other relevant dependencies still need inventory.
|
|
||||||
|
|
||||||
## Discoverability and Release Integration
|
|
||||||
|
|
||||||
End README with the project-license and attribution statements prescribed by the
|
|
||||||
OSS policy, using Audita's actual grant. `COPYRIGHT` owns the separately licensed
|
|
||||||
policy scope; README links to that grant and the attribution document.
|
|
||||||
|
|
||||||
Update `docs/release.md` to specify the final `COPYRIGHT | LICENSE | ATTRIBUTION`
|
|
||||||
links in every new release note and hosted download page. Use absolute Gitea URLs
|
|
||||||
at the release's immutable tag or source commit. Reuse the checked-in note as the
|
|
||||||
hosted body where supported; verify the publication plugin's supported mechanism
|
|
||||||
and make the necessary focused `.woodpecker/release.yml` change. Avoid separately
|
|
||||||
maintained release prose or notice lists.
|
|
||||||
|
|
||||||
Preserve the current atomic source/tag publication approach, supported targets,
|
|
||||||
and completion point. Align procedure wording with upstream's preparation,
|
|
||||||
ambiguous-push recovery, and asynchronous retry rules. Handle completed-roadmap
|
|
||||||
retirement during preparation, preserving planned/deferred work and repairing
|
|
||||||
links; this roadmap update does not authorize deleting other roadmaps now.
|
|
||||||
|
|
||||||
Check intended future-tag paths against the candidate source without requiring
|
|
||||||
the tag to resolve before publication. Apply the formatting convention
|
|
||||||
prospectively; leave published tags, notes, artifacts, and checksums intact.
|
|
||||||
Concrete historical omissions, if discovered, receive deliberate treatment under
|
|
||||||
their applicable terms and release correction rules.
|
|
||||||
|
|
||||||
## Work Required and Acceptance Criteria
|
|
||||||
|
|
||||||
Implementation covers the seven policy documents, `AGENTS.md`,
|
|
||||||
`docs/development.md`, `COPYRIGHT`, `docs/oss/attribution.md`, applicable files in
|
|
||||||
`docs/oss/licenses/`, README, the release procedure, and focused hosted-body
|
|
||||||
publication wiring. Update this roadmap's implementation status when appropriate.
|
|
||||||
The [implementation plan](implementation.md) owns the editing sequence and
|
|
||||||
verification steps.
|
|
||||||
|
|
||||||
The work is complete when:
|
|
||||||
|
|
||||||
- Shared policy language matches the selected upstream revision after required
|
|
||||||
template removal and customization; retained variations have substantive
|
|
||||||
Audita-specific reasons as identified above.
|
|
||||||
- All seven policies have accurate versions, adoption dates, and file-specific
|
|
||||||
commit-pinned source links. Public discovery links and selective orientation
|
|
||||||
are in place, with no leftover template-only instructions or broken links.
|
|
||||||
- Copied policy/guidance scope, modification status, credit, CC-BY-SA text, and
|
|
||||||
root grant notice follow the documented upstream arrangement while preserving
|
|
||||||
Audita's application license.
|
|
||||||
- Relevant components have verified House Format entries, exact source/notice
|
|
||||||
commits, and complete applicable license texts, without editorial commentary.
|
|
||||||
- README and future release documentation provide the required navigation, and
|
|
||||||
hosted release content comes from the checked-in note through a verified
|
|
||||||
publication mechanism.
|
|
||||||
- Audita's application contracts, architectural decisions, platform commitments,
|
|
||||||
release completion semantics, and published history are preserved.
|
|
||||||
- Affected text, links, license copies, metadata, and publication configuration
|
|
||||||
pass proportionate review and validation. No release is needed solely to test
|
|
||||||
adoption; behavioral workflow changes receive focused validation if required.
|
|
||||||
|
|
||||||
## Scope Boundaries and Open Questions
|
|
||||||
|
|
||||||
No policy decisions remain open for the agreed scope. Component-specific notices
|
|
||||||
and the publication plugin's supported release-body mechanism require factual
|
|
||||||
verification during implementation. Raise a decision only for a concrete
|
|
||||||
unresolved obligation or material tradeoff uncovered by that work.
|
|
||||||
|
|
||||||
This feature does not require application refactoring, dependency upgrades,
|
|
||||||
mandatory scanners, a full SBOM, recurring legal approvals, synchronization
|
|
||||||
checks, additional human release gates, publication-time network checks, or CI
|
|
||||||
waiting. The documentation convention alone does not require downloadable
|
|
||||||
license assets, source bundles, or a different binary package format.
|
|
||||||
@@ -1,161 +0,0 @@
|
|||||||
# Maintainability Audit: Findings and Recommended Changes
|
|
||||||
|
|
||||||
**Reviewed:** 2026-09-25, source commit `011b021`.
|
|
||||||
**Scope:** Recent partial-decision reconciliation, validation context, prompt
|
|
||||||
alignment, and request-local reference changes, including related existing code.
|
|
||||||
**Status:** R27–R30 addressed; see the resolution record below. Previous findings and resolution records remain in Git
|
|
||||||
history through `011b021`.
|
|
||||||
|
|
||||||
## Architecture Assessment
|
|
||||||
|
|
||||||
The recent changes fit Audita's existing boundaries. Proposal generation,
|
|
||||||
validation, prompt assets, and outcome reporting already have suitable owners;
|
|
||||||
this audit does not recommend a new common package or a broad refactor.
|
|
||||||
|
|
||||||
The strongest opportunities are shared prompt instructions and request-scoped
|
|
||||||
validation context preparation. Two smaller improvements concern recovery-reason
|
|
||||||
classification and partial-result assembly; both predate the latest commits.
|
|
||||||
These are bounded maintenance recommendations, not new release gates or changes
|
|
||||||
to correction policy, response acceptance, or reporting semantics.
|
|
||||||
|
|
||||||
## R27 — P2: Consolidate Shared Prompt Instructions
|
|
||||||
|
|
||||||
**Category:** Repeated behavioral-contract text; risk of future drift.
|
|
||||||
|
|
||||||
**Evidence:** The four [proposal prompts](../../internal/prompts/assets/prompts/modules.grammar.proposal.yaml)
|
|
||||||
repeat literal source-span matching, request-local identity, replacement-text,
|
|
||||||
source-based proposal, and confidence instructions. The three active
|
|
||||||
[validator prompts](../../internal/prompts/assets/prompts/validators.editorial_review.yaml)
|
|
||||||
repeat candidate-reference, supplemental-context, and verdict-confidence guidance.
|
|
||||||
Recent contract changes required coordinated edits across these assets. The
|
|
||||||
[prompt registry](../../internal/prompts/registry.go) already provides a shared
|
|
||||||
hardening fragment supplied through the role-bound adapter.
|
|
||||||
|
|
||||||
**Recommended change:** Extract genuinely common passages into embedded fragments
|
|
||||||
under `internal/prompts/assets/shared`, following the existing fragment-supply
|
|
||||||
approach. Keep the fragments owned by `internal/prompts` and use the existing
|
|
||||||
PromptKit rendering path. Keep module-specific examples, replacement-policy
|
|
||||||
instructions, protected-name exceptions, and semantic criteria in their individual
|
|
||||||
prompt definitions. Do not merge similar-looking rules with different meanings.
|
|
||||||
|
|
||||||
**Acceptance:** Common instructions have one maintained source and appear in all
|
|
||||||
intended rendered prompts. The existing module and validator distinctions remain
|
|
||||||
intact, including glossary replace-all behavior and the spoken-word protected-name
|
|
||||||
exception. Focused rendering checks cover fragment inclusion and representative
|
|
||||||
module-specific guidance without requiring exact-prose snapshots or introducing
|
|
||||||
a separate templating framework.
|
|
||||||
|
|
||||||
## R28 — P2: Prepare Validation Context Once Per Request
|
|
||||||
|
|
||||||
**Category:** Repeated computation and overlapping segment lookup.
|
|
||||||
|
|
||||||
**Evidence:** [BuildLLMValidationRequest](../../internal/framework/validators/llm_validators.go)
|
|
||||||
already indexes transcript segments, but
|
|
||||||
[validationContext](../../internal/framework/validators/llm_context.go) scans the
|
|
||||||
transcript again for each candidate. It also calls
|
|
||||||
[NewProtectedVocabulary](../../internal/framework/validators/protected_terms.go)
|
|
||||||
for each candidate, rebuilding the same vocabulary and compiling its matching
|
|
||||||
expression repeatedly. This is duplicated preparation rather than copied code.
|
|
||||||
|
|
||||||
**Recommended change:** Prepare a segment-position lookup and protected vocabulary
|
|
||||||
once per validation request, then pass that prepared context into the existing
|
|
||||||
context-selection helper. Keep the implementation within `framework/validators`.
|
|
||||||
Continue selecting glossary evidence for each candidate individually because its
|
|
||||||
proposed replacement can mention different terms. A process-wide cache or new
|
|
||||||
context-management package is unnecessary for this cleanup.
|
|
||||||
|
|
||||||
**Acceptance:** Candidates in one validation request reuse the prepared lookup and
|
|
||||||
vocabulary. Context still consists of the available immediate neighbors within the
|
|
||||||
current chunk and relevant glossary entries in glossary order. Candidate-specific
|
|
||||||
replacement text still participates in glossary selection. Existing optional-context
|
|
||||||
trimming, token estimation, and oversized-input handling retain their behavior.
|
|
||||||
Verify these outcomes with focused behavioral tests rather than assertions about
|
|
||||||
private helper call counts or unmeasured performance guarantees.
|
|
||||||
|
|
||||||
## R29 — P3: Share Execution-Recovery Classification
|
|
||||||
|
|
||||||
**Category:** Duplicate reporting rule; predates the latest commits.
|
|
||||||
|
|
||||||
**Evidence:** [Proposal generation](../../internal/framework/proposal_generation/generate.go)
|
|
||||||
and [LLM validation](../../internal/framework/validators/llm_validators.go)
|
|
||||||
independently translate structural-repair and transient-retry counts into
|
|
||||||
`output_repaired` and `service_recovered` reasons. The conversion is the same in
|
|
||||||
both locations; the surrounding success and domain-outcome decisions differ.
|
|
||||||
|
|
||||||
**Recommended change:** Place a small conversion helper in the existing
|
|
||||||
`framework/outcomes` package and use it from both callers. Keep the decision to
|
|
||||||
record recoveries at the call sites, along with domain-specific reasons such as
|
|
||||||
filtered validator decisions. Do not combine proposal and validation orchestration
|
|
||||||
or move domain-success classification into this helper.
|
|
||||||
|
|
||||||
**Acceptance:** Both callers use the same execution-count conversion. Existing
|
|
||||||
reason ordering, successful-operation reporting, and domain-specific additions
|
|
||||||
remain intact. A partially unavailable validation batch continues to record its
|
|
||||||
failure and execution counts without acquiring recovery records merely because a
|
|
||||||
repair or retry occurred. Existing outcome tests should continue to protect these
|
|
||||||
distinctions.
|
|
||||||
|
|
||||||
## R30 — P3: Centralize Partial Validator Result Assembly
|
|
||||||
|
|
||||||
**Category:** Repeated early-return construction; predates the latest commits.
|
|
||||||
|
|
||||||
**Evidence:** Several early returns in
|
|
||||||
[LLMBackedValidator.Validate](../../internal/framework/validators/llm_validators.go)
|
|
||||||
repeat nested slice appends to combine immediate decisions, oversized-input
|
|
||||||
decisions, and decisions from completed model batches, together with accumulated
|
|
||||||
operation outcomes. This makes it easier for a future failure path to omit facts
|
|
||||||
already collected by the validator.
|
|
||||||
|
|
||||||
**Recommended change:** Use a small local result-building helper to assemble the
|
|
||||||
partial result consistently. Keep error handling, batching, success-path ordering,
|
|
||||||
and cardinality enforcement explicit. A general-purpose result-aggregation package
|
|
||||||
is not needed.
|
|
||||||
|
|
||||||
**Acceptance:** Cancellation and failure paths retain the same previously collected
|
|
||||||
decisions and operations. Result ordering and cardinality behavior remain unchanged;
|
|
||||||
the helper does not enforce complete cardinality on interrupted work. Retain the
|
|
||||||
existing partial-progress and fatal-failure coverage, adding tests only for a
|
|
||||||
material gap exposed by the refactor.
|
|
||||||
|
|
||||||
## Boundaries to Preserve
|
|
||||||
|
|
||||||
- Keep proposal and validator reference mapping separate. Proposals can target the
|
|
||||||
same segment repeatedly; duplicate validator decisions make that candidate
|
|
||||||
unavailable. Similar indexing arithmetic alone does not justify a generic
|
|
||||||
reference-management abstraction.
|
|
||||||
- Keep distinct protected-term policy checks separate. They already share the
|
|
||||||
vocabulary implementation and intentionally apply different policies.
|
|
||||||
- Preserve exact source-span matching, request-local references, batching, and
|
|
||||||
partial-success behavior. Consolidation is not a reason to strengthen contracts
|
|
||||||
or broaden the accepted correction scope.
|
|
||||||
|
|
||||||
## Review Verification and Limits
|
|
||||||
|
|
||||||
- This is a source-level maintainability audit, not a finding of demonstrated
|
|
||||||
production failures or a measured performance assessment.
|
|
||||||
- No application tests were rerun for the original read-only audit; its roadmap
|
|
||||||
update used documentation link and whitespace checks. Implementation validation
|
|
||||||
is recorded below.
|
|
||||||
- Future remediation should run focused behavioral tests and the broader suite
|
|
||||||
appropriate to the actual change, following the testing policy. No live-provider
|
|
||||||
evaluation or new evaluation framework is required by these recommendations.
|
|
||||||
- The findings above describe the audited baseline. Implementation and validation
|
|
||||||
are recorded below; no release publication is included.
|
|
||||||
|
|
||||||
## Resolution Record
|
|
||||||
|
|
||||||
Implemented as focused changes within the existing packages. Proposal and validator
|
|
||||||
reference mapping and distinct protected-term policies remain separate.
|
|
||||||
|
|
||||||
| Finding | Resolution and evidence |
|
|
||||||
| --- | --- |
|
|
||||||
| R27 | Common proposal and validation instructions live in two embedded shared fragments, supplied through the existing role-bound PromptKit adapter. Individual YAML definitions retain policy differences and examples. Rendering tests check fragment inclusion once per applicable prompt and representative module-specific guidance. |
|
|
||||||
| R28 | Validation request construction builds one segment-position lookup and protected vocabulary, reused by context selection for every candidate. A new test verifies that two replacements of the same source span receive their own relevant glossary entries. Existing neighbor, glossary-order, chunk-boundary, and budget tests pass. |
|
|
||||||
| R29 | Both proposal generation and validation use `outcomes.ExecutionRecoveries`. Call sites still control success and domain-specific recovery reasons. Existing tests cover combined recoveries and failed validation with repair counts but no recovery record. |
|
|
||||||
| R30 | A local result builder combines collected decisions and operation outcomes for partial returns and completed validation. Sorting and full cardinality checks remain outside it. Existing fatal-batch, partial-progress, runner, and reporting tests pass. |
|
|
||||||
|
|
||||||
Validation: `go test ./...` passed, including PromptKit preparation, partial-decision
|
|
||||||
reconciliation, subprocess reports, and fatal-progress coverage. Documentation
|
|
||||||
links and whitespace checks passed. No live model evaluation or performance
|
|
||||||
measurement was performed; the changes make no new model-quality or latency
|
|
||||||
promise.
|
|
||||||
@@ -1,290 +0,0 @@
|
|||||||
# Resilient Processing and Focused Run Reports
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
**Status:** Implemented. This document records the selected behavior and user
|
|
||||||
intent. Concrete design decisions and completion evidence are in
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md).
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Audita should reliably produce a useful transcript even when its LLM services
|
|
||||||
cannot improve it. Exhausted retries or unusable model responses must not turn
|
|
||||||
an otherwise processable transcript into a failed production run.
|
|
||||||
|
|
||||||
> Failure of every LLM operation across every stage and chunk still produces a
|
|
||||||
> successful output containing the deterministically normalized transcript, with
|
|
||||||
> no LLM corrections applied. Partial success preserves as many fully validated,
|
|
||||||
> applicable corrections as possible.
|
|
||||||
|
|
||||||
Deterministic normalization remains intentional and may change segment layout,
|
|
||||||
identifiers, and text before LLM work begins. Output uses the selected transcript
|
|
||||||
encoder; byte-for-byte preservation of the original input and mapping corrections
|
|
||||||
back to pre-normalization segmentation are not requirements.
|
|
||||||
|
|
||||||
The overriding goal is to minimize how much useful processing is lost because of
|
|
||||||
one problematic transcript segment. A stage/chunk work unit is an attribution
|
|
||||||
boundary, not an all-or-nothing transaction.
|
|
||||||
|
|
||||||
Existing reports contain useful information, but there are no compatibility
|
|
||||||
obligations for reporting or debug output. Reuse existing machinery only when it
|
|
||||||
makes the implementation simpler and clearer; preservation is not itself a goal.
|
|
||||||
Internal structures and measurements must serve Audita's processing workflow.
|
|
||||||
The reporting package adapts those outcomes to structured and human-facing
|
|
||||||
presentations, rather than reshaping processing around a reporting convenience.
|
|
||||||
|
|
||||||
## Historical Baseline and Implemented Change
|
|
||||||
|
|
||||||
The [pipeline](../internal/pipeline.md), [LLM integration](../integrations/openai-compatible-llm.md),
|
|
||||||
[report assembly](../internal/diagnostics-reporting.md), and
|
|
||||||
[artifact lifecycle](../operations.md) describe current behavior.
|
|
||||||
|
|
||||||
Before this work, exhausted structural repairs could become warnings and skipped
|
|
||||||
proposals or rejected validation batches. Other model runtime failures canceled
|
|
||||||
peer work, stopped subsequent modules, and prevented final transcript output.
|
|
||||||
Successful repairs appeared in execution observations rather than a dedicated
|
|
||||||
recovery classification. Detailed artifacts were written during ordinary runs
|
|
||||||
and could later be removed by retention. The working transcript was re-chunked
|
|
||||||
between modules.
|
|
||||||
|
|
||||||
This feature changed those behaviors. It builds on the completed
|
|
||||||
[library adoption](library-adoption.md) and [review remediation](remediation.md).
|
|
||||||
Fatal cancellation remains necessary; unsuccessful model work must be separated
|
|
||||||
from failures that actually require cancellation of the run.
|
|
||||||
|
|
||||||
## Fixed Chunks and Ordered Stages
|
|
||||||
|
|
||||||
Normalize first, then partition the transcript once. Preserve chunk boundaries,
|
|
||||||
segment membership, segment identity, and ordering throughout LLM processing.
|
|
||||||
Text evolves as accepted corrections are applied. Recombine chunks in original
|
|
||||||
order after the LLM stages finish.
|
|
||||||
|
|
||||||
A stage is a configured module instance. Repeated instances of the same module
|
|
||||||
have distinct identities. Proposal generation and each validator are operations
|
|
||||||
within a stage; validation may use multiple batches within a chunk. A work unit
|
|
||||||
is one stage instance applied to one chunk.
|
|
||||||
|
|
||||||
Preserve the overall stage sequence: finish a stage across its chunks before
|
|
||||||
advancing to the next stage. Concurrent chunk work shares PromptKit's backend
|
|
||||||
capacity. Failure in one unit does not prevent other chunks or later stages
|
|
||||||
from executing.
|
|
||||||
|
|
||||||
Retain operator control over chunk size and the current defaults. Production
|
|
||||||
experience indicates that chunk size affects model latency nonlinearly and that
|
|
||||||
useful settings are well below backend context limits. Tuning is an empirical
|
|
||||||
operator decision. Do not add automatic headroom calculations, cross-profile
|
|
||||||
sizing, adaptive repartitioning, or automatic tuning in this feature. A later
|
|
||||||
operation that cannot process its input follows the same safe error handling;
|
|
||||||
it does not silently alter chunk boundaries. Validation batching within a chunk
|
|
||||||
remains supported.
|
|
||||||
|
|
||||||
Independent whole-pipeline traversal by chunks and domain-aware response repair
|
|
||||||
are explicitly deferred in [future work](future.md). Read-only neighboring
|
|
||||||
context, cross-chunk edits, and transcript-wide restructuring are outside this
|
|
||||||
unit, without becoming permanent prohibitions on future capabilities.
|
|
||||||
|
|
||||||
## Partial Success and Failure Isolation
|
|
||||||
|
|
||||||
- Keep bounded transport retries, structural repairs, operation deadlines, and
|
|
||||||
shared backend capacity. Completion comes from safe fallback, not unlimited
|
|
||||||
attempts or an outer retry loop around PromptKit.
|
|
||||||
- Apply a correction only after it passes every required validator and the
|
|
||||||
application checks. Failed validation must never be treated as approval.
|
|
||||||
- If a validation batch fails, exclude its affected candidates while allowing
|
|
||||||
other candidates to continue through the remaining validators. Preserve fully
|
|
||||||
validated, applicable corrections from successful batches in the same unit.
|
|
||||||
- Track candidates by stable proposal identity. Batch membership can change
|
|
||||||
between validators; equal batch numbers do not establish candidate identity.
|
|
||||||
- A failed proposal-generation operation leaves its chunk unchanged for that
|
|
||||||
stage because there are no reliable proposals to salvage. Failures later in
|
|
||||||
validation need not discard unrelated successful candidates.
|
|
||||||
- Preserve earlier stages' corrections and other chunks' successful work.
|
|
||||||
Record partial success explicitly; a unit can apply corrections and have errors.
|
|
||||||
- Continue despite model-operation failures, including exhausted repairs/retries,
|
|
||||||
unusable model decisions, provider/transport failures, local operation deadlines,
|
|
||||||
and admission failures. An operation need not exhaust its attempt budget to fail.
|
|
||||||
- A valid empty proposal list or valid assessments rejecting all corrections is
|
|
||||||
successful processing. Correction count alone does not determine failure.
|
|
||||||
- Keep safe filtering of defective proposals. Do not retry ordinary editorial
|
|
||||||
rejections. For now, an unusable validator decision set excludes the affected
|
|
||||||
candidates and records an error after existing permitted recovery. New bounded
|
|
||||||
feedback/regeneration for domain-invalid responses belongs to future work.
|
|
||||||
- Retain fatal treatment for invalid input/configuration, internal defects,
|
|
||||||
explicit run cancellation, and inability to produce/write the transcript.
|
|
||||||
Caller cancellation is distinct from a local model-operation timeout.
|
|
||||||
- Completed runs with model errors return success. Required report-write failures
|
|
||||||
are an additional fatal output condition when reporting is enabled, as below.
|
|
||||||
|
|
||||||
PromptKit owns transport, transient retries, structural repair, and admission.
|
|
||||||
Audita owns domain acceptance, partial outcomes, fallback, and classification.
|
|
||||||
ConfigKit owns configuration resolution. Debug and reporting choices must not
|
|
||||||
change correction decisions, retry budgets, or continuation.
|
|
||||||
|
|
||||||
## Errors, Recoveries, and Debug
|
|
||||||
|
|
||||||
Classify by eventual outcome and impact. These are three reporting categories;
|
|
||||||
debug is evidence, not a failure severity. Whether execution continues is a
|
|
||||||
separate property.
|
|
||||||
|
|
||||||
| Category | Meaning | Representative cases |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Errors | Intended work could not complete reliably, or a fatal run-level operation failed. Affected candidates are excluded; other corrections can succeed. | Exhausted repair; unavailable provider; unusable validation decisions; operation deadline. |
|
|
||||||
| Recoveries | A problem was handled successfully through retries, repair, or safe filtering. No operator intervention is implied. | Invalid output repaired; transient service failure recovered; defective proposals safely discarded. |
|
|
||||||
| Debug | Detailed evidence of processing and ordinary decisions. | Proposals, applied corrections, validator decisions, prompts/responses, individual attempts, execution traces. |
|
|
||||||
|
|
||||||
Normal editorial disagreement belongs in debug. A validator correctly rejecting
|
|
||||||
a meaning-changing edit is expected behavior. A proposal referring to nonexistent
|
|
||||||
original text is defective; safely filtering it is a recovery. An unresolved
|
|
||||||
missing/duplicate decision-index response is an error, not a recovery.
|
|
||||||
|
|
||||||
Record an operation's eventual outcome once. Several failed attempts followed by
|
|
||||||
success produce a consolidated recovery. Attempts ending in failure contribute
|
|
||||||
to error details, not additional recoveries. Earlier operations that recovered
|
|
||||||
remain recorded if a later operation introduces an error in the same unit.
|
|
||||||
|
|
||||||
Separate output-quality recovery, defective-proposal filtering, and service
|
|
||||||
recovery with structured reasons. Network instability must not look like a prompt
|
|
||||||
defect. Do not introduce an arbitrary warning threshold or a fourth category.
|
|
||||||
|
|
||||||
### Counting and Processing Volume
|
|
||||||
|
|
||||||
- Count error-bearing work units once per stage/chunk, consolidating underlying
|
|
||||||
operation failures as details. A partially successful unit still counts once.
|
|
||||||
- Count recoveries once per recovered operation. Aggregate defective-proposal
|
|
||||||
filtering per proposal response, retaining affected-proposal counts as detail.
|
|
||||||
Attempts and individual discarded proposals are not independent incidents.
|
|
||||||
- Keep work-unit coverage separate from operation/recovery counts. Show processing
|
|
||||||
volume with explicitly labeled denominators; do not divide recovered-operation
|
|
||||||
counts by work-unit counts and call that a recovery rate.
|
|
||||||
- Report fatal run-level errors separately from the stage/chunk grid when they
|
|
||||||
cannot be attributed to a work unit. Do not invent a chunk to accommodate them.
|
|
||||||
- Derive YAML records, Markdown summaries, and CLI counts from actual processing
|
|
||||||
outcomes with deterministic ordering, rather than independently calculating them.
|
|
||||||
|
|
||||||
## Production Outputs, Placement, and Lifecycle
|
|
||||||
|
|
||||||
Reporting is enabled by default. The ordinary bundle is:
|
|
||||||
|
|
||||||
- `errors.yml`: structured error-bearing units, their underlying failures, and
|
|
||||||
applicable run-level failures.
|
|
||||||
- `recoveries.yml`: structured recovered operations and pertinent counts.
|
|
||||||
- `report.md`: a concise human-facing summary of outcome, corrections, processing
|
|
||||||
volume, errors, recoveries, and their locations in the pipeline.
|
|
||||||
|
|
||||||
Write all three documents, including empty event lists for uneventful completed
|
|
||||||
runs. Use Audita-owned schemas with safe explanations and stable reason codes;
|
|
||||||
retain run, stage-instance, chunk, operation, and validator/batch identities where
|
|
||||||
applicable. The implementation plan specifies the versioned schemas; neither old
|
|
||||||
report formats nor raw library structures define the new output contract.
|
|
||||||
|
|
||||||
Place reports beside the output transcript. Retain transcript-to-stdout support
|
|
||||||
with an explicit report-directory selection, defaulting to the invocation working
|
|
||||||
directory. Preserve the existing overwrite convention: a new run can replace files
|
|
||||||
at its selected destinations. Do not add a mandatory fresh-directory restriction
|
|
||||||
or compatibility exports.
|
|
||||||
|
|
||||||
Before model work, create initial versions of all three reports carrying a common
|
|
||||||
run identifier and explicit `in_progress` status (rendered as prose in Markdown).
|
|
||||||
Fail immediately if initialization fails. Also check the transcript destination;
|
|
||||||
creating neighboring reports does not prove that an existing transcript file is
|
|
||||||
writable. Preflight must not prematurely destroy an existing transcript.
|
|
||||||
|
|
||||||
Finalize all required outputs and check every final write. Initial write success
|
|
||||||
cannot guarantee later success if space or permissions change. Failure to write
|
|
||||||
the transcript or finalize a required report returns nonzero and must not erase
|
|
||||||
an already produced transcript. Where possible, finalize available reports with
|
|
||||||
an explicit failed status on fatal termination. Abrupt termination may leave
|
|
||||||
`in_progress`; elaborate crash-recovery machinery is not required.
|
|
||||||
|
|
||||||
Provide an option to disable reporting entirely. It suppresses all three files
|
|
||||||
(`errors.yml`, `recoveries.yml`, and `report.md`), including initialization and
|
|
||||||
final writes, while retaining the brief CLI summary and independently selectable
|
|
||||||
debug output. Disabled reporting must not create or require a writable report
|
|
||||||
destination, and must not remove reports from earlier runs.
|
|
||||||
Transcript output failures remain fatal regardless of reporting selection.
|
|
||||||
|
|
||||||
Ordinary reports survive independently of debug cleanup. Reporting configuration
|
|
||||||
and finalization must not depend on retaining a debug run directory.
|
|
||||||
|
|
||||||
## Debug and Internal Boundaries
|
|
||||||
|
|
||||||
Production issue records are compact. Full responses, transcript excerpts,
|
|
||||||
correction lists, and complete validation decisions are opt-in debug evidence.
|
|
||||||
Use a single ConfigKit-backed debug boolean with a `--debug` CLI override. Debug
|
|
||||||
artifacts may use JSON where useful. Preserve configured-secret redaction in every
|
|
||||||
presentation.
|
|
||||||
|
|
||||||
Debug-off prevents detailed artifact persistence; writing and later deleting
|
|
||||||
those files is insufficient. In-memory domain state needed to process candidates
|
|
||||||
correctly is not debug output. Debug and ordinary reporting are independent
|
|
||||||
selections and do not alter processing behavior.
|
|
||||||
|
|
||||||
There is no obligation to retain `--report-json`, old artifact layouts, parallel
|
|
||||||
exports, or old retention settings. Reuse code only where useful to the new
|
|
||||||
workflow. Enabled debug output is retained on success or failure; disabled debug
|
|
||||||
creates no debug directory. This supersedes the retention question in the
|
|
||||||
[runtime follow-up](runtime-behavior.md). The legacy `never` behavior is not the
|
|
||||||
meaning of the new reporting-disable option. Optional debug-write failures produce
|
|
||||||
a safe stderr diagnostic without changing correction decisions or the run exit
|
|
||||||
status; ordinary report and transcript-write failures retain their fatal semantics.
|
|
||||||
|
|
||||||
## Deterministic Human-Facing Presentation
|
|
||||||
|
|
||||||
Use the CLI style:
|
|
||||||
|
|
||||||
> Completed: 18 corrections applied; 3 work units with errors; 7 recoveries.
|
|
||||||
|
|
||||||
Do not say that all error-bearing units were skipped: some apply valid corrections.
|
|
||||||
Include report locations when reporting is enabled and the files exist. Send the
|
|
||||||
summary to stderr so stdout remains exclusively available for transcript output.
|
|
||||||
Disabled reporting does not suppress the summary or fatal-error messages.
|
|
||||||
|
|
||||||
Keep `report.md` concise and high-level. Use separate error and recovery matrices
|
|
||||||
with stages as columns and chunks as rows. Include every chunk in fixed order;
|
|
||||||
retain configured stage order and distinct labels for repeated module instances.
|
|
||||||
No automatic width detection, thresholds, adaptive omission, or table splitting.
|
|
||||||
A wide or long table is acceptable.
|
|
||||||
|
|
||||||
The error matrix counts error-bearing work units: each attempted cell is 0 or 1.
|
|
||||||
Recovery cells count recovered operations and can exceed 1. An error cell may
|
|
||||||
coexist with recoveries and successfully applied corrections in the same unit.
|
|
||||||
Distinguish completed work with zero events from unattempted work; do not portray
|
|
||||||
canceled/unavailable processing as clean completion. Include totals, processing
|
|
||||||
volume, and a short reason breakdown. Detailed inspection belongs in YAML or debug.
|
|
||||||
|
|
||||||
## Target Acceptance Criteria
|
|
||||||
|
|
||||||
- Total LLM failure still returns success and the encoded normalized transcript,
|
|
||||||
provided required output writes succeed.
|
|
||||||
- Mixed-success runs preserve independently validated, applicable corrections
|
|
||||||
within the same stage/chunk as a failed batch. No candidate bypasses a required
|
|
||||||
validator; later validators handle survivors by stable proposal identity.
|
|
||||||
- Subsequent stages and unaffected chunks execute after recoverable failures.
|
|
||||||
Fatal errors and explicit cancellation stop and join active work appropriately.
|
|
||||||
- Chunk boundaries and identity remain fixed, with operator-selected sizing and
|
|
||||||
unchanged defaults; final recombination preserves ordering.
|
|
||||||
- Valid no-change/editorial-rejection outcomes are successful. Repaired/filtered
|
|
||||||
output and unresolved failures receive distinct classifications and counts.
|
|
||||||
- YAML, Markdown, and CLI agree on partial outcomes and totals; error matrices
|
|
||||||
count affected work units, recovery matrices count recovered operations.
|
|
||||||
- With reporting enabled, all three reports are initialized before model work,
|
|
||||||
finalized and retained, including empty lists. Initial or final required-output
|
|
||||||
write failures produce nonzero exits without destroying completed transcript output.
|
|
||||||
- Reporting can be disabled without requiring its destination. Debug can be enabled
|
|
||||||
independently; neither selection changes processing outcomes. Debug-off writes
|
|
||||||
no detailed debug artifacts, and stdout transcript output remains supported.
|
|
||||||
- Reports use the fixed deterministic layout without adaptive presentation logic.
|
|
||||||
- Meaningful controlled-provider tests cover exhausted recovery, failed validation
|
|
||||||
batches with successful survivors, total failure, fatal cancellation, output
|
|
||||||
failures, and reporting/debug selections. Preserve domain safeguards, capacity
|
|
||||||
limits, and redaction.
|
|
||||||
- Canonical CLI, configuration, operations, integration, internal, and architecture
|
|
||||||
documentation describes the implemented final behavior.
|
|
||||||
|
|
||||||
## Implementation Reference
|
|
||||||
|
|
||||||
The product decisions are settled. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md) defines
|
|
||||||
the selected settings and precedence, YAML schemas, counts and ordering, interrupted
|
|
||||||
work presentation, destination checks, and output finalization. It also owns the
|
|
||||||
ordered implementation stages and validation work. This document remains the
|
|
||||||
reference for the desired behavior and its acceptance criteria.
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
# Runtime Behavior Decisions
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-24
|
|
||||||
|
|
||||||
The documentation migration identified existing behaviors for separate product
|
|
||||||
consideration. The resilience implementation resolved the diagnostics-retention
|
|
||||||
question; the input-timestamp question remains open.
|
|
||||||
|
|
||||||
- Resolved: legacy `auto`/`always`/`never` retention settings were removed.
|
|
||||||
Debug is explicitly enabled with `diagnostics.debug` or `--debug`, creates no
|
|
||||||
directory when disabled, and retains its directory on every outcome when enabled.
|
|
||||||
- Resolved: [configuration](../config.md) now treats explicit zero as disabling
|
|
||||||
transient retries or structural repair independently; PromptKit performs both.
|
|
||||||
- Decide whether missing or null input timestamps should be rejected; current
|
|
||||||
decoding is documented in the [input contract](../integrations/transcript-glossary-files.md).
|
|
||||||
|
|
||||||
These decisions are independent of the completed documentation migration and
|
|
||||||
release workflow.
|
|
||||||
@@ -1,203 +0,0 @@
|
|||||||
# Consistent Span Deletion
|
|
||||||
|
|
||||||
**Last updated:** 2026-09-25
|
|
||||||
|
|
||||||
**Status:** Implemented. This document owns the desired behavior and user intent.
|
|
||||||
The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md)
|
|
||||||
records the concrete steps, selected implementation details, and completion
|
|
||||||
evidence.
|
|
||||||
|
|
||||||
## Purpose and User Intent
|
|
||||||
|
|
||||||
Align proposal prompts, the LLM response schema, internal validation, and
|
|
||||||
application behavior so legitimate text-span deletions can be considered and
|
|
||||||
applied through the ordinary correction workflow. An empty replacement should
|
|
||||||
not trigger structural repair merely because it is empty, nor invalidate other
|
|
||||||
usable corrections returned alongside it.
|
|
||||||
|
|
||||||
This is a consistency fix within Audita's current editing scope. It does not
|
|
||||||
authorize whole-segment removal, broaden module responsibilities, or introduce
|
|
||||||
new recovery machinery. The accepted [resilience behavior](resilience.md) remains
|
|
||||||
in effect, including partial success, fixed chunks, and validation before mutation.
|
|
||||||
|
|
||||||
## Historical Baseline (`0b6cd98`)
|
|
||||||
|
|
||||||
The source reviewed at `0b6cd98` contains a mismatch:
|
|
||||||
|
|
||||||
- The [spoken-word prompt](../../internal/prompts/assets/prompts/modules.spoken_word.proposal.yaml)
|
|
||||||
permits removing fillers and asks for the exact affected span and replacement,
|
|
||||||
without explaining how to represent deletion. Grammar permits punctuation and
|
|
||||||
spacing changes, which can also naturally involve deletion.
|
|
||||||
- The shared [correction response schema](../../internal/framework/responseschema/assets/correction_set.v1.json)
|
|
||||||
requires `corrected_text` to have at least one character. Every built-in proposal
|
|
||||||
module uses that schema. An empty replacement invalidates the response before
|
|
||||||
Audita evaluates individual proposals.
|
|
||||||
- PromptKit performs bounded structural repair. If repair does not produce a
|
|
||||||
valid response, the proposal operation fails and yields no candidates, including
|
|
||||||
otherwise valid proposals in the same response. The current resilience path
|
|
||||||
records the error and continues other work.
|
|
||||||
- [CorrectionProposal](../../internal/framework/proposals/proposal.go) already
|
|
||||||
allows empty replacement text. The
|
|
||||||
[preview/application logic](../../internal/framework/proposals/preview.go)
|
|
||||||
supports literal span deletion when the resulting segment remains nonblank.
|
|
||||||
- Domain shape checks currently reject whitespace-only original spans, although
|
|
||||||
the response schema permits them and grammar prompts permit spacing cleanup.
|
|
||||||
This is another restriction to align with the target contract below.
|
|
||||||
- The validator named `non_empty_corrected_text` actually checks the resulting
|
|
||||||
segment, not the replacement string. Both this check and application prevent
|
|
||||||
a segment from becoming empty or whitespace-only. Application changes segment
|
|
||||||
text; it does not remove segment records.
|
|
||||||
|
|
||||||
A temporary controlled-provider review reproduced whole-response rejection for
|
|
||||||
an empty replacement alongside a valid correction, and immediate schema acceptance
|
|
||||||
for a space-only replacement. Existing proposal tests confirm internal support for
|
|
||||||
partial deletion and rejection of blank resulting segments. These observations
|
|
||||||
explain a plausible production failure mechanism; they do not establish the cause
|
|
||||||
of any particular historical incident without its payload and running version.
|
|
||||||
|
|
||||||
## Target Correction Contract
|
|
||||||
|
|
||||||
- `original_text` identifies an existing, nonempty literal span in the current
|
|
||||||
target segment. Whitespace-only spans are eligible; an empty string identifies
|
|
||||||
no source span and remains invalid. Matching and ambiguity checks still apply
|
|
||||||
under the module's replacement policy: `require_unique` for grammar, spoken-word,
|
|
||||||
and homophones; `replace_all` for glossary.
|
|
||||||
- `corrected_text` remains a required string. An empty string means delete the
|
|
||||||
identified span; it does not mean remove the segment record or skip validation.
|
|
||||||
Missing fields and non-string values remain structurally invalid.
|
|
||||||
- After applying a correction, the segment must contain non-whitespace text.
|
|
||||||
Enforce this using the source segment and replacement result in Audita, not a
|
|
||||||
minimum replacement length in the structural schema.
|
|
||||||
- Every accepted deletion passes the same required validators as other edits in
|
|
||||||
its module, including confidence, protected terms, editorial review, and meaning
|
|
||||||
preservation where applicable. Empty replacement is neither automatic approval
|
|
||||||
nor an automatic rejection.
|
|
||||||
- Application preserves segment identity, speaker, timing, membership, ordering,
|
|
||||||
and fixed chunk boundaries. Recheck applicability against text produced by
|
|
||||||
earlier accepted corrections, retaining existing conflict handling.
|
|
||||||
- Preserve literal replacement behavior. Prompts should select spans and
|
|
||||||
replacements that handle adjacent whitespace and punctuation appropriately;
|
|
||||||
this feature does not add automatic trimming or a second normalization pass.
|
|
||||||
Neither side of a correction is rejected solely for containing only whitespace.
|
|
||||||
For example, reducing a uniquely identified run of two spaces to one space may
|
|
||||||
be appropriate; validity still depends on the resulting text and domain checks.
|
|
||||||
|
|
||||||
Preserve the intended safeguards and outcomes, not every detail of the current
|
|
||||||
implementation. Validator ordering, internal structure, and payload construction
|
|
||||||
may change when needed to achieve this consistent contract. Existing behavior or
|
|
||||||
a passing test does not justify retaining a conflicting restriction.
|
|
||||||
|
|
||||||
Illustrative cases:
|
|
||||||
|
|
||||||
| Segment | Original span | Replacement | Intended handling |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `uh hello` | `uh ` | empty string | Structurally valid; may produce `hello` after required validation. |
|
|
||||||
| `uh hello` | `uh hello` | `hello` | Remains a valid representation of the same cleanup, subject to validation. |
|
|
||||||
| `hello world` | two spaces | one space | Eligible spacing correction, subject to unique matching and required validation. |
|
|
||||||
| `uh` | `uh` | empty string | Structurally valid proposal, then filtered because the resulting segment would be blank. |
|
|
||||||
| `hello world` | `hello world` | spaces only | Filtered because the resulting segment would be blank. |
|
|
||||||
|
|
||||||
These examples describe eligibility and mechanical results, not unconditional
|
|
||||||
editorial approval. An unchanged segment or valid empty correction list remains
|
|
||||||
a successful no-change outcome.
|
|
||||||
|
|
||||||
## Prompt and Module Alignment
|
|
||||||
|
|
||||||
Review relevant prompt definitions for consistency and edit only those needing
|
|
||||||
clarification. Where needed, explain empty-string deletion, whitespace-only spans,
|
|
||||||
and the resulting-segment limit with a concise, module-appropriate example. The
|
|
||||||
model should omit a correction that would leave a blank segment rather than invent
|
|
||||||
padding or replacement content. Consistent guidance is the goal; adding deletion
|
|
||||||
language to every prompt is not a requirement.
|
|
||||||
|
|
||||||
Retain each module's existing scope:
|
|
||||||
|
|
||||||
- Spoken-word cleanup can propose appropriate filler or dysfluency deletion while
|
|
||||||
preserving substantive content and intentional emphasis.
|
|
||||||
- Grammar can propose permitted punctuation or spacing deletion; allowing an
|
|
||||||
empty replacement does not authorize word removal or spoken-word cleanup.
|
|
||||||
- Glossary and homophone modules retain their existing correction purposes.
|
|
||||||
Shared deletion syntax does not grant them new editorial responsibilities.
|
|
||||||
|
|
||||||
Review the active LLM-validator prompts and their input construction as well as
|
|
||||||
proposal prompts. Validators must understand the before/after effect of deletion
|
|
||||||
and assess it under their existing criteria. A correct negative editorial decision
|
|
||||||
remains ordinary processing, not a defect that should be regenerated until approved.
|
|
||||||
|
|
||||||
The resulting-segment validator's name and documented identity should describe
|
|
||||||
that responsibility accurately, rather than imply that the replacement string
|
|
||||||
itself must be nonempty. Preserve the existing blank-result failure reason and
|
|
||||||
safeguard while making that deliberate identity change; current chain position
|
|
||||||
and preview implementation are not independent product requirements.
|
|
||||||
|
|
||||||
## Outcomes and Reporting
|
|
||||||
|
|
||||||
A legitimate deletion is an ordinary correction. Empty replacement alone must
|
|
||||||
produce neither structural repair nor a recovery classification. If another
|
|
||||||
problem required successful repair or filtering, retain that actual recovery.
|
|
||||||
|
|
||||||
A schema-valid proposal that would blank its stage-input segment is a defective
|
|
||||||
proposal handled through existing safe filtering. Preserve the
|
|
||||||
`empty_resulting_segment` reason
|
|
||||||
and consolidated `proposals_filtered` recovery behavior; other usable proposals
|
|
||||||
in the response continue through required validation. Ordinary editorial rejection
|
|
||||||
remains debug evidence rather than a model error or filtering recovery.
|
|
||||||
|
|
||||||
A stage can propose several edits to one segment in the same response. Each edit
|
|
||||||
is evaluated against the stage input, then approved edits are applied sequentially.
|
|
||||||
If a proposal was applicable to that input but an earlier accepted edit makes it
|
|
||||||
unsafe, skip it as an apply-time conflict. Do not count that conflict as a defective
|
|
||||||
proposal, model error, or filtering recovery. For example, in `"uh um"`, deleting
|
|
||||||
`"uh "` and deleting `"um"` each leaves text individually; after the first deletion,
|
|
||||||
the second must be skipped to prevent a blank result. Retain the skipped decision
|
|
||||||
in debug evidence. The existing runner already distinguishes this situation.
|
|
||||||
|
|
||||||
Debug evidence must accurately retain the empty replacement and the corresponding
|
|
||||||
validation/application outcome. Ordinary reports retain their existing counts and
|
|
||||||
purpose; no new report schema, incident category, or deletion-specific counter is
|
|
||||||
needed. Unrelated structural defects may still exhaust PromptKit repair and produce
|
|
||||||
an error for the operation, as today.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A controlled valid response containing an empty replacement reaches Audita's
|
|
||||||
domain validation without structural repair solely due to that empty string.
|
|
||||||
- A permitted span deletion can pass the required validators and reach the final
|
|
||||||
transcript while preserving segment metadata and fixed chunk membership.
|
|
||||||
- A mixed response can apply an approved deletion and another approved correction;
|
|
||||||
an empty replacement does not invalidate the whole response.
|
|
||||||
- A proposal that would blank its stage-input segment is filtered independently,
|
|
||||||
retains the existing reason, and does not prevent unrelated valid candidates
|
|
||||||
from succeeding. A proposal made unsafe only by an earlier accepted edit is
|
|
||||||
skipped without adding an error or filtering recovery; a blank result is still
|
|
||||||
prevented and the conflict remains visible in debug evidence.
|
|
||||||
- A uniquely matching whitespace-only original span can reach validation and
|
|
||||||
application. Empty originals remain invalid, ambiguous matches remain subject
|
|
||||||
to module policy, and whitespace-only resulting segments remain disallowed.
|
|
||||||
- Protected-term and meaning-changing deletions remain subject to their normal
|
|
||||||
rejection rules. No candidate bypasses required validation.
|
|
||||||
- A deletion needing no recovery counts as an ordinary applied correction; actual
|
|
||||||
repair/filtering and editorial rejection retain their existing classifications.
|
|
||||||
Debug evidence represents the empty replacement faithfully.
|
|
||||||
- Schema and domain tests distinguish an empty string from missing/non-string
|
|
||||||
replacement values and distinguish empty replacement from blank resulting text.
|
|
||||||
- Prompt guidance and validator naming express the same contract as the code.
|
|
||||||
Tests protect outcomes rather than exact prompt prose or incidental formatting.
|
|
||||||
|
|
||||||
Use deterministic, offline controlled responses for behavioral integration tests;
|
|
||||||
no live-provider or paid evaluation is a prerequisite. Representative production
|
|
||||||
examples can inform prompt quality without expanding this into an evaluation
|
|
||||||
framework or guaranteeing that models always follow the instructions.
|
|
||||||
|
|
||||||
## Deferred Work
|
|
||||||
|
|
||||||
[Whole-segment removal](future.md#whole-segment-removal) is only potential future
|
|
||||||
work and first requires deciding whether its benefit warrants implementation.
|
|
||||||
Domain-aware regeneration and independent whole-pipeline traversal also remain
|
|
||||||
in [future.md](future.md). This feature changes neither retry budgets nor the
|
|
||||||
Audita/PromptKit ownership boundary and requires no upstream library feature.
|
|
||||||
|
|
||||||
Implementation sequencing and concrete names belong to the
|
|
||||||
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md).
|
|
||||||
The target remains consistent deletion of spans within surviving segments, with
|
|
||||||
no new operator controls or report types.
|
|
||||||
82
examples/README.md
Normal file
82
examples/README.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# Maintained examples
|
||||||
|
|
||||||
|
Build Audita from the repository root with Go 1.27 or newer. The commands below
|
||||||
|
assume the executable is on `PATH`. Each project selects its main file from its
|
||||||
|
own directory, which also determines companion resources and overlay context.
|
||||||
|
All models, credentials, names, and endpoints are illustrative.
|
||||||
|
|
||||||
|
## Local processing
|
||||||
|
|
||||||
|
From `examples/local`, edit `prompts.yml` to match a reachable OpenAI-compatible
|
||||||
|
local service and its model. Then run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config config.yml
|
||||||
|
audita process --config config.yml --output corrected.json transcript.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Processing makes model requests. Reports are written beside the selected output
|
||||||
|
unless disabled. For an offline validator preview, from the same directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prepared audita/validators.meaning_reversal_review \
|
||||||
|
--config config.yml --profile local \
|
||||||
|
--var "PayloadJSON=$(cat validation-candidate.json)" \
|
||||||
|
--var 'TranscriptDescriptionBlock='
|
||||||
|
```
|
||||||
|
|
||||||
|
To preview the repair instruction and its output schema without inference:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prepared audita/modules.grammar.repair \
|
||||||
|
--config config.yml --profile local \
|
||||||
|
--var "RepairRequestJSON=$(cat repair-candidate.json)"
|
||||||
|
```
|
||||||
|
|
||||||
|
These previews render supplied data without inference. The native command help
|
||||||
|
and [CLI reference](../docs/cli.md) describe additional inspection and validation.
|
||||||
|
|
||||||
|
## Service and overlay
|
||||||
|
|
||||||
|
From `examples/service`, edit the synthetic endpoint and models in `prompts.yml`.
|
||||||
|
Set `AUDITA_EXAMPLE_KEY` to your service credential before a real request. Inspect
|
||||||
|
without a provider call:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config config.yml --overlay testing
|
||||||
|
audita promptkit show profile service-test --config config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
The example does not promise a provider's model availability, price, or quality.
|
||||||
|
|
||||||
|
## Edit a prompt and run again
|
||||||
|
|
||||||
|
From `examples/prompt-edit`, inspect the operator replacement:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita promptkit show prompt audita/modules.grammar.proposal --config config.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit its message in `prompts.yml`, repeat the inspection, and run `audita process
|
||||||
|
--config config.yml --output corrected.json transcript.json` against a configured
|
||||||
|
local service. Each invocation captures the files anew.
|
||||||
|
|
||||||
|
## Shared glossary library
|
||||||
|
|
||||||
|
From `examples/campaigns`, `first.yml` and `second.yml` each consume the common
|
||||||
|
`shared.yml` through the companion `glossary.yml`. Inspect each configuration:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita config validate --config first.yml
|
||||||
|
audita config validate --config second.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
With a local service configured, process either campaign:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
audita process --config first.yml --output first-output.json transcript.json
|
||||||
|
audita process --config second.yml --output second-output.json transcript.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit `prompts.yml` for a local service before processing either campaign. The
|
||||||
|
first uses separate party and spell corrections; the second selects only party.
|
||||||
8
examples/campaigns/first.yml
Normal file
8
examples/campaigns/first.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm: {proposal_profile: local}
|
||||||
|
glossaries: [party, spells]
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- {id: names, module: glossary, glossaries: [party]}
|
||||||
|
- {id: magic, module: glossary, glossaries: [spells]}
|
||||||
|
- {module: grammar}
|
||||||
4
examples/campaigns/glossary.yml
Normal file
4
examples/campaigns/glossary.yml
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
config:
|
||||||
|
imports:
|
||||||
|
- path: shared.yml
|
||||||
|
required: true
|
||||||
5
examples/campaigns/prompts.yml
Normal file
5
examples/campaigns/prompts.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
7
examples/campaigns/second.yml
Normal file
7
examples/campaigns/second.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
version: 3
|
||||||
|
llm: {proposal_profile: local}
|
||||||
|
glossaries: [party]
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- {id: names, module: glossary}
|
||||||
|
- {module: grammar}
|
||||||
12
examples/campaigns/shared.yml
Normal file
12
examples/campaigns/shared.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: lyra
|
||||||
|
name: Lyra
|
||||||
|
category: character
|
||||||
|
description: A fictional adventurer.
|
||||||
|
spells:
|
||||||
|
entries:
|
||||||
|
- id: shield
|
||||||
|
name: Shield
|
||||||
|
category: spell
|
||||||
|
aliases: [Ward]
|
||||||
9
examples/campaigns/transcript.json
Normal file
9
examples/campaigns/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "Lyra raises a ward before the journey."
|
||||||
|
}
|
||||||
|
]
|
||||||
8
examples/local/config.yml
Normal file
8
examples/local/config.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party]
|
||||||
7
examples/local/glossary.yml
Normal file
7
examples/local/glossary.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: jesters
|
||||||
|
name: Jesters
|
||||||
|
category: faction
|
||||||
|
description: A traveling group of performers.
|
||||||
|
aliases: [Jester]
|
||||||
5
examples/local/prompts.yml
Normal file
5
examples/local/prompts.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
17
examples/local/repair-candidate.json
Normal file
17
examples/local/repair-candidate.json
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"proposal_ref": 1,
|
||||||
|
"correction": {
|
||||||
|
"id": 1,
|
||||||
|
"original_text": "uh",
|
||||||
|
"corrected_text": ""
|
||||||
|
},
|
||||||
|
"original_segment_text": "uh hello",
|
||||||
|
"corrected_segment_text": " hello",
|
||||||
|
"findings": [
|
||||||
|
{
|
||||||
|
"validator": "editorial_review",
|
||||||
|
"reason": "llm_rejected",
|
||||||
|
"explanation": "Removing only 'uh' leaves unwanted leading whitespace."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
9
examples/local/transcript.json
Normal file
9
examples/local/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
1
examples/local/validation-candidate.json
Normal file
1
examples/local/validation-candidate.json
Normal file
@@ -0,0 +1 @@
|
|||||||
|
[{"correction_index":1,"speaker":"Narrator","original_text":"not open","corrected_text":"open","original_segment_text":"The door is not open.","corrected_segment_text":"The door is open."}]
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
version: 2
|
|
||||||
backends:
|
|
||||||
local:
|
|
||||||
endpoint: http://127.0.0.1:11434/v1
|
|
||||||
concurrency: 1
|
|
||||||
llm:
|
|
||||||
proposal_profile: local-proposal
|
|
||||||
validation_profile: local-validation
|
|
||||||
pipeline:
|
|
||||||
modules: [grammar]
|
|
||||||
output:
|
|
||||||
schema: bare-segments
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
llm:
|
|
||||||
proposal_profile: service-proposal-testing
|
|
||||||
validation_profile: service-validation-testing
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
version: 2
|
|
||||||
backends:
|
|
||||||
service:
|
|
||||||
endpoint: https://openrouter.ai/api/v1
|
|
||||||
credential: AUDITA_SERVICE_KEY
|
|
||||||
concurrency: 2
|
|
||||||
queue_capacity: 1024
|
|
||||||
llm:
|
|
||||||
proposal_profile: service-proposal
|
|
||||||
validation_profile: service-validation
|
|
||||||
transient_retries: 2
|
|
||||||
repair_attempts: 2
|
|
||||||
operation_timeout: 15m
|
|
||||||
pipeline:
|
|
||||||
modules: [glossary, homophones, glossary, spoken_word, grammar]
|
|
||||||
output:
|
|
||||||
schema: audita-v1
|
|
||||||
chunking:
|
|
||||||
# Requires at least this many normalized segments; omit for automatic chunking.
|
|
||||||
target_sections: 8
|
|
||||||
max_section_tokens: 8192
|
|
||||||
min_section_tokens: 2048
|
|
||||||
normalization:
|
|
||||||
max_segment_gap: 4.0
|
|
||||||
ellipsis_gap: 3.5
|
|
||||||
max_segment_duration: 60.0
|
|
||||||
max_segment_tokens: 2048
|
|
||||||
thresholds:
|
|
||||||
glossary: 0.8
|
|
||||||
homophones: 0.8
|
|
||||||
spoken_word: 0.8
|
|
||||||
grammar: 0.8
|
|
||||||
context:
|
|
||||||
description: "General context for domain vocabulary and speaker names."
|
|
||||||
reporting:
|
|
||||||
enabled: true
|
|
||||||
stdout_dir: .
|
|
||||||
diagnostics:
|
|
||||||
debug: false
|
|
||||||
work_dir: /tmp/audita
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: local-proposal
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: local-validation
|
|
||||||
backend: local
|
|
||||||
model: replace-with-local-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-proposal-testing
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-proposal
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 2048
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-validation-testing
|
|
||||||
backend: service
|
|
||||||
model: replace-with-testing-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: service-validation
|
|
||||||
backend: service
|
|
||||||
model: replace-with-provider-model
|
|
||||||
temperature: 0
|
|
||||||
max_tokens: 1024
|
|
||||||
top_p: 1
|
|
||||||
timeout_seconds: 120
|
|
||||||
8
examples/prompt-edit/config.yml
Normal file
8
examples/prompt-edit/config.yml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: local
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party]
|
||||||
7
examples/prompt-edit/glossary.yml
Normal file
7
examples/prompt-edit/glossary.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
party:
|
||||||
|
entries:
|
||||||
|
- id: jesters
|
||||||
|
name: Jesters
|
||||||
|
category: faction
|
||||||
|
description: A traveling group of performers.
|
||||||
|
aliases: [Jester]
|
||||||
23
examples/prompt-edit/prompts.yml
Normal file
23
examples/prompt-edit/prompts.yml
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
profiles:
|
||||||
|
- id: local
|
||||||
|
endpoint: http://localhost:11434/v1
|
||||||
|
model: example-model
|
||||||
|
api_key_required: false
|
||||||
|
collections:
|
||||||
|
audita:
|
||||||
|
prompts:
|
||||||
|
- id: modules.grammar.proposal
|
||||||
|
messages:
|
||||||
|
- role: system
|
||||||
|
content: Correct only clearly supported grammatical errors.
|
||||||
|
- role: user
|
||||||
|
content: |
|
||||||
|
{{ .ProposalContract }}
|
||||||
|
{{ .HardeningPolicy }}
|
||||||
|
{{ .TranscriptDescriptionBlock }}
|
||||||
|
Glossary: {{ .GlossaryJSON }}
|
||||||
|
Transcript: {{ .SectionJSON }}
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: correction_set.v1.json
|
||||||
9
examples/prompt-edit/transcript.json
Normal file
9
examples/prompt-edit/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
7
examples/service/config.yml
Normal file
7
examples/service/config.yml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
version: 3
|
||||||
|
llm:
|
||||||
|
proposal_profile: service-proposal
|
||||||
|
validation_profile: service-review
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: grammar
|
||||||
3
examples/service/overlays/testing.yml
Normal file
3
examples/service/overlays/testing.yml
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
llm:
|
||||||
|
proposal_profile: service-test
|
||||||
|
validation_profile: service-test
|
||||||
16
examples/service/prompts.yml
Normal file
16
examples/service/prompts.yml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
backends:
|
||||||
|
- id: service
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
api_key_env: AUDITA_EXAMPLE_KEY
|
||||||
|
concurrency_limit: 2
|
||||||
|
queue_capacity: 8
|
||||||
|
profiles:
|
||||||
|
- id: service-proposal
|
||||||
|
backend: service
|
||||||
|
model: example-proposal-model
|
||||||
|
- id: service-review
|
||||||
|
backend: service
|
||||||
|
model: example-review-model
|
||||||
|
- id: service-test
|
||||||
|
extends: {id: service-proposal}
|
||||||
|
model: example-testing-model
|
||||||
9
examples/service/transcript.json
Normal file
9
examples/service/transcript.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"speaker": "Narrator",
|
||||||
|
"start": 0,
|
||||||
|
"end": 2,
|
||||||
|
"text": "The gestures gather beside the stage."
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
glossary:
|
|
||||||
- name: Audita
|
|
||||||
aliases:
|
|
||||||
- audita
|
|
||||||
category: product
|
|
||||||
summary: The Audita transcript correction CLI.
|
|
||||||
31
go.mod
31
go.mod
@@ -1,20 +1,25 @@
|
|||||||
module gitea.maximumdirect.net/eric/audita
|
module gitea.maximumdirect.net/eric/audita/v4
|
||||||
|
|
||||||
go 1.25.5
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2
|
go.maximumdirect.net/ckconfig v0.9.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
go.maximumdirect.net/ckprompt v0.10.0
|
||||||
|
go.maximumdirect.net/commandkit v0.7.0
|
||||||
|
go.maximumdirect.net/configkit v1.2.1
|
||||||
|
go.maximumdirect.net/promptkit v0.23.2
|
||||||
|
go.maximumdirect.net/releasekit v0.10.0
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 // indirect
|
github.com/spf13/cobra v1.10.2 // indirect
|
||||||
github.com/kr/pretty v0.3.1 // indirect
|
github.com/spf13/pflag v1.1.0 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
github.com/yuin/goldmark/v2 v2.1.16 // indirect
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
golang.org/x/mod v0.43.0 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
golang.org/x/text v0.44.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tool go.maximumdirect.net/releasekit/cmd/releasekit
|
||||||
|
|||||||
62
go.sum
62
go.sum
@@ -1,33 +1,35 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2 h1:ZMxmv8pOnLtAx2hFTf8Z+pHKUbR89xErkBhwjr4yK24=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.4.2/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2 h1:2RqI+KtflvSh7vLQaTAQCmMaFQscEA7EPwPazyvXRoY=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.11.2/go.mod h1:onm/FBPUKQ8/NSPwTEk26Ut+dVX2iXcq8vEseB+g/0E=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
|
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
|
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
|
||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/spf13/pflag v1.1.0 h1:6W6MbVUrnMb8PEhjkqwsVLY7ryuzRfFHF4mQd7L94ag=
|
||||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
github.com/spf13/pflag v1.1.0/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
github.com/yuin/goldmark/v2 v2.1.16 h1:MXvCVo+Cb2SVJ5cwJUzXetDjI7HECRJCkcqylEQtcJo=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/yuin/goldmark/v2 v2.1.16/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
go.maximumdirect.net/ckconfig v0.9.0 h1:s9Ikga4wt4K1NpLCX9bLhm5vbcm1Fgqy9yk2lmO4Sss=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
go.maximumdirect.net/ckconfig v0.9.0/go.mod h1:yGjFVr2UzjJjiwDdMwsKehuQfvvtv7rTEOIk5NQOY00=
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
go.maximumdirect.net/ckprompt v0.10.0 h1:t2UyWSke+bWH9aTDrw5/bteD+TO0y5LD0NWw0y2KniQ=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
go.maximumdirect.net/ckprompt v0.10.0/go.mod h1:vx8lZnelASctQtyxY2yV4PuTZfLpE9EqDa+LPj3+SHM=
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
go.maximumdirect.net/commandkit v0.7.0 h1:toLDGhHoVe6gBX5HWuzQ/Joz0cBqEhJXolW21m3axBk=
|
||||||
|
go.maximumdirect.net/commandkit v0.7.0/go.mod h1:sVp3JXKpm2YrP4Ie8Q3nWmQgwBdPN8uO+wQU72erDns=
|
||||||
|
go.maximumdirect.net/configkit v1.2.1 h1:f+/6WjNl6CULbMr6ZSVwi4aty9rmuDfJlvpeU7I+HZM=
|
||||||
|
go.maximumdirect.net/configkit v1.2.1/go.mod h1:ewu6blSokUDCMovy8JBXAdXRikGmlHXAQpFghnPWRfs=
|
||||||
|
go.maximumdirect.net/promptkit v0.23.2 h1:nE+Qqqbw5GN4PHD2e5OEWUHXcYkESZm3ryZVBsKWwFQ=
|
||||||
|
go.maximumdirect.net/promptkit v0.23.2/go.mod h1:vPFg76VXcffP0JBFMS2WADCHwV5FYvO2O4bvIoiDlwc=
|
||||||
|
go.maximumdirect.net/releasekit v0.10.0 h1:3GOgaIigXTkvYaRkntWZASE//2NTR1+IhkoKmt64aJo=
|
||||||
|
go.maximumdirect.net/releasekit v0.10.0/go.mod h1:cT5NZNEcf7SKDx4MPG40YNK9V4bNyIxcLLFuNlbGQ90=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0 h1:/LG1pNO84Z4l603j/xMn69CcS5OYhMQ/f7l2LoQUGuw=
|
||||||
|
go.yaml.in/yaml/v4 v4.0.0/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||||
|
golang.org/x/mod v0.43.0 h1:MFuq5J6s35W9D3IjkwyWc9vHh0nENOrt5K8Iof4ofmg=
|
||||||
|
golang.org/x/mod v0.43.0/go.mod h1:4ec1JnuyBz5+FIVhg86UsA39YG8BpnrZdJYSNHmGzHY=
|
||||||
|
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||||
|
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
|
|||||||
110
internal/cli/app_wiring.go
Normal file
110
internal/cli/app_wiring.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/prompts"
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
"go.maximumdirect.net/ckprompt"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
"go.maximumdirect.net/releasekit/buildinfo"
|
||||||
|
)
|
||||||
|
|
||||||
|
type invocationState struct {
|
||||||
|
engines []engineEntry
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
type engineEntry struct {
|
||||||
|
selection ckconfig.Selection
|
||||||
|
engine *pk.Engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadApplication(ctx context.Context, selection ckconfig.Selection, _ struct{}, _ *invocationState) (*ck.Result[config.Config], error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var source ck.Source
|
||||||
|
if selection.Config.Set {
|
||||||
|
if strings.TrimSpace(selection.Config.Value) == "" {
|
||||||
|
return nil, fmt.Errorf("--config requires a nonempty path")
|
||||||
|
}
|
||||||
|
source = ck.File(selection.Config.Value)
|
||||||
|
} else {
|
||||||
|
source = ck.SearchConfig("audita", ck.Searches(ck.WorkingDirectory(), ck.UserConfigDirectory(), ck.UnixSystemDirectories()))
|
||||||
|
}
|
||||||
|
return config.Load(ck.Input{Source: source}, selection.Overlay.Value, selection.Overrides...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
loaded, err := loadApplication(ctx, selection, input, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var client *http.Client
|
||||||
|
if state != nil {
|
||||||
|
client = state.client
|
||||||
|
}
|
||||||
|
return llm.NewEngine(ctx, loaded, client)
|
||||||
|
}
|
||||||
|
|
||||||
|
func capturePromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
if state != nil {
|
||||||
|
for _, entry := range state.engines {
|
||||||
|
if reflect.DeepEqual(entry.selection, selection) {
|
||||||
|
return entry.engine, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
engine, err := loadPromptEngine(ctx, selection, input, state)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if state != nil {
|
||||||
|
state.engines = append(state.engines, engineEntry{selection: selection, engine: engine})
|
||||||
|
}
|
||||||
|
return engine, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildApplication(extra ...commandkit.Declaration) (*commandkit.App, error) {
|
||||||
|
return buildApplicationWithClient(nil, extra...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildApplicationWithClient(client *http.Client, extra ...commandkit.Declaration) (*commandkit.App, error) {
|
||||||
|
appScope := ckconfig.FromConfigKit("app", "Audita application settings.", loadApplication)
|
||||||
|
appScope.Features.Overrides = true
|
||||||
|
promptScope := ckprompt.ConfigScope(loadPromptEngine)
|
||||||
|
configCommand := ckconfig.Command(ckconfig.Suite[struct{}, *invocationState]{NewState: func() *invocationState { return &invocationState{client: client} }, Scopes: []ckconfig.Scope[struct{}, *invocationState]{appScope, promptScope}})
|
||||||
|
definitions := ckprompt.WithNamespaces(ckprompt.FromEngine(capturePromptEngine), pk.Namespaces{})
|
||||||
|
definitions.NewState = func() *invocationState { return &invocationState{client: client} }
|
||||||
|
definitions = ckprompt.WithRequestValidation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.ValidationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
|
||||||
|
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
|
||||||
|
})
|
||||||
|
definitions = ckprompt.WithRequestPreparation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.PreparationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
|
||||||
|
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
|
||||||
|
})
|
||||||
|
definitions = ckprompt.WithSmokeTesting(definitions, capturePromptEngine, nil)
|
||||||
|
declarations := []commandkit.Declaration{configCommand, commandkit.Completion("audita"), processCommand()}
|
||||||
|
declarations = append(declarations, ckprompt.Declarations(definitions)...)
|
||||||
|
declarations = append(declarations, extra...)
|
||||||
|
return commandkit.New(commandkit.Identity{Name: "audita", Version: buildinfo.Version()}, declarations...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// applicationPromptRequest supplies instructions only for Audita's maintained prompts.
|
||||||
|
func applicationPromptRequest(request pk.RunRequest, namespaces *pk.Namespaces) pk.RunRequest {
|
||||||
|
id := request.PromptID
|
||||||
|
if !strings.Contains(id, "/") && namespaces != nil && namespaces.Prompt != "" {
|
||||||
|
id = namespaces.Prompt + "/" + id
|
||||||
|
}
|
||||||
|
if _, ok := prompts.LookupMetadata(id); ok {
|
||||||
|
request.Vars = prompts.WithSharedVariables(request.Vars)
|
||||||
|
}
|
||||||
|
return request
|
||||||
|
}
|
||||||
232
internal/cli/app_wiring_test.go
Normal file
232
internal/cli/app_wiring_test.go
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
"go.maximumdirect.net/ckprompt"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNativeConfigurationAndResourceCommands(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(file, []byte("version: 3\ncontext:\n description: example\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
args []string
|
||||||
|
contains string
|
||||||
|
}{
|
||||||
|
{[]string{"--help"}, "audita"},
|
||||||
|
{[]string{"version"}, "audita"},
|
||||||
|
{[]string{"process", "--help"}, "transcript"},
|
||||||
|
{[]string{"config", "show", "--scope", "app", "--config", file}, "example"},
|
||||||
|
{[]string{"promptkit", "list", "prompts", "--config", file}, "audita/modules.grammar.proposal"},
|
||||||
|
} {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status != 0 || !strings.Contains(out.String(), tc.contains) {
|
||||||
|
t.Errorf("%v: status=%d out=%q err=%q", tc.args, result.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNativeConfigurationSelectionsAndComparison(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
|
||||||
|
for name, data := range map[string]string{first: "context:\n description: before\n", second: "context:\n description: after\n"} {
|
||||||
|
if err := os.WriteFile(name, []byte(data), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
outcome := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--scope=app", "--config", first, "--against-config", second, "--set=/context/description=\"left\"", "--against-set=/context/description=\"right\""}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status != 3 || !strings.Contains(out.String(), "left") || !strings.Contains(out.String(), "right") {
|
||||||
|
t.Fatalf("status=%d output=%q err=%q", outcome.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
stderr.Reset()
|
||||||
|
outcome = app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config", filepath.Join(dir, "missing.yml")}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status == 0 {
|
||||||
|
t.Fatalf("missing file accepted: %s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExplicitEmptyConfigurationPathFails(t *testing.T) {
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config="}, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status == 0 {
|
||||||
|
t.Fatal("explicit empty path accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPromptConfigurationComparisonAcquiresEqualSelectionsIndependently(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
main := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resource := filepath.Join(dir, "prompts.yml")
|
||||||
|
writeProfile := func(model string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(resource, []byte("profiles: [{id: local, model: "+model+"}]\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeProfile("before")
|
||||||
|
scope := ckprompt.ConfigScope(func(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
|
||||||
|
engine, err := loadPromptEngine(ctx, selection, input, state)
|
||||||
|
writeProfile("after")
|
||||||
|
return engine, err
|
||||||
|
})
|
||||||
|
side := ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{
|
||||||
|
Config: commandkit.Supplied[string]{Value: main, Set: true},
|
||||||
|
}}
|
||||||
|
result, err := scope.Compare(context.Background(), ckconfig.ComparisonRequest[struct{}]{Left: side, Right: side}, &invocationState{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, difference := range result.Value {
|
||||||
|
if strings.Contains(difference.Left.YAML, "before") && strings.Contains(difference.Right.YAML, "after") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("comparison did not capture the model before and after the resource changed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
resources := `backends:
|
||||||
|
- id: local
|
||||||
|
endpoint: https://example.invalid/v1
|
||||||
|
smoke_tests:
|
||||||
|
default_profile: fixture
|
||||||
|
authentication:
|
||||||
|
kind: none
|
||||||
|
profiles:
|
||||||
|
- id: fixture
|
||||||
|
backend: local
|
||||||
|
model: sample
|
||||||
|
collections:
|
||||||
|
audita:
|
||||||
|
prompts:
|
||||||
|
- id: modules.grammar.proposal
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}'
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
calls := 0
|
||||||
|
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
body := `{}`
|
||||||
|
if req.Method == http.MethodPost {
|
||||||
|
requestBody, err := io.ReadAll(req.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
content := "PROMPTKIT_OK"
|
||||||
|
if strings.Contains(string(requestBody), "response_format") {
|
||||||
|
content = `{"status":"ok"}`
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(content)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}`
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
||||||
|
})}
|
||||||
|
app, err := buildApplicationWithClient(client)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
execute := func(args ...string) (commandkit.Outcome, string, string) {
|
||||||
|
t.Helper()
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
return result, out.String(), stderr.String()
|
||||||
|
}
|
||||||
|
common := []string{"--config", file, "--llm-profile", "fixture"}
|
||||||
|
args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...)
|
||||||
|
args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction")
|
||||||
|
outcome, out, stderr := execute(args...)
|
||||||
|
if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") {
|
||||||
|
t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...)
|
||||||
|
args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
||||||
|
outcome, out, stderr = execute(args...)
|
||||||
|
if outcome.Status != 0 {
|
||||||
|
t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...)
|
||||||
|
validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
|
||||||
|
outcome, out, stderr = execute(validatorPreview...)
|
||||||
|
if outcome.Status != 0 || !strings.Contains(out, "teh cat") {
|
||||||
|
t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr)
|
||||||
|
}
|
||||||
|
if calls != 0 {
|
||||||
|
t.Fatalf("inspection made %d provider calls", calls)
|
||||||
|
}
|
||||||
|
outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local")
|
||||||
|
if outcome.Status != 0 || calls == 0 {
|
||||||
|
t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
||||||
|
}
|
||||||
|
previous := calls
|
||||||
|
outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json")
|
||||||
|
if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") {
|
||||||
|
t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
|
||||||
|
|
||||||
|
func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) {
|
||||||
|
request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}}
|
||||||
|
result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"})
|
||||||
|
if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" {
|
||||||
|
t.Fatalf("variables=%v", result.Vars)
|
||||||
|
}
|
||||||
|
if _, changed := request.Vars["ProposalContract"]; changed {
|
||||||
|
t.Fatal("input variables mutated")
|
||||||
|
}
|
||||||
|
generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}}
|
||||||
|
result = applicationPromptRequest(generic, nil)
|
||||||
|
if len(result.Vars) != 1 || result.Vars["x"] != "y" {
|
||||||
|
t.Fatalf("generic request changed: %v", result.Vars)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
var systemConfigPaths = []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"}
|
|
||||||
|
|
||||||
type configInput struct {
|
|
||||||
ConfigPath string
|
|
||||||
ConfigPathSet bool
|
|
||||||
Overlay string
|
|
||||||
View string
|
|
||||||
TranscriptPath string
|
|
||||||
GlossaryPath string
|
|
||||||
OutputPath string
|
|
||||||
Overrides []ck.Override
|
|
||||||
}
|
|
||||||
|
|
||||||
type selectedConfig struct {
|
|
||||||
Path string
|
|
||||||
Source string
|
|
||||||
Result *ck.Result[config.Config]
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseConfigInput owns the common setting flags and process-only path flags.
|
|
||||||
// A transcript before flags and one after flags are both accepted.
|
|
||||||
func parseConfigInput(command string, args []string, output io.Writer) (configInput, bool, error) {
|
|
||||||
var input configInput
|
|
||||||
fs := flag.NewFlagSet(command, flag.ContinueOnError)
|
|
||||||
fs.SetOutput(output)
|
|
||||||
configPath := fs.String("config", "", "Main configuration YAML file")
|
|
||||||
overlay := fs.String("overlay", "", "Selected configuration overlay")
|
|
||||||
if command == "print-effective" {
|
|
||||||
fs.String("view", "settings", "Effective view: settings, profiles, or explain")
|
|
||||||
}
|
|
||||||
settingPaths := map[string]ck.Path{
|
|
||||||
"proposal-profile": ck.Keys("llm", "proposal_profile"),
|
|
||||||
"validation-profile": ck.Keys("llm", "validation_profile"),
|
|
||||||
"output-schema": ck.Keys("output", "schema"),
|
|
||||||
"validation-max-prompt-tokens": ck.Keys("validation", "max_prompt_tokens"),
|
|
||||||
"target-sections": ck.Keys("chunking", "target_sections"),
|
|
||||||
"max-section-tokens": ck.Keys("chunking", "max_section_tokens"),
|
|
||||||
"min-section-tokens": ck.Keys("chunking", "min_section_tokens"),
|
|
||||||
"normalize-max-segment-gap": ck.Keys("normalization", "max_segment_gap"),
|
|
||||||
"normalize-ellipsis-gap": ck.Keys("normalization", "ellipsis_gap"),
|
|
||||||
"normalize-max-segment-duration": ck.Keys("normalization", "max_segment_duration"),
|
|
||||||
"normalize-max-segment-tokens": ck.Keys("normalization", "max_segment_tokens"),
|
|
||||||
"glossary-confidence-threshold": ck.Keys("thresholds", "glossary"),
|
|
||||||
"homophones-confidence-threshold": ck.Keys("thresholds", "homophones"),
|
|
||||||
"spoken-word-confidence-threshold": ck.Keys("thresholds", "spoken_word"),
|
|
||||||
"grammar-confidence-threshold": ck.Keys("thresholds", "grammar"),
|
|
||||||
"transcript-description": ck.Keys("context", "description"),
|
|
||||||
"reporting": ck.Keys("reporting", "enabled"),
|
|
||||||
"report-dir": ck.Keys("reporting", "stdout_dir"),
|
|
||||||
"debug": ck.Keys("diagnostics", "debug"),
|
|
||||||
"work-dir": ck.Keys("diagnostics", "work_dir"),
|
|
||||||
"modules": ck.Keys("pipeline", "modules"),
|
|
||||||
}
|
|
||||||
for name := range settingPaths {
|
|
||||||
switch name {
|
|
||||||
case "reporting", "debug":
|
|
||||||
fs.Bool(name, false, "Setting override")
|
|
||||||
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
|
|
||||||
fs.Int(name, 0, "Setting override")
|
|
||||||
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
|
|
||||||
fs.Float64(name, 0, "Setting override")
|
|
||||||
default:
|
|
||||||
fs.String(name, "", "Setting override")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if command == "process" {
|
|
||||||
fs.String("glossary", "", "Glossary YAML file")
|
|
||||||
fs.String("output", "", "Corrected transcript output file")
|
|
||||||
}
|
|
||||||
parseArgs := args
|
|
||||||
front := ""
|
|
||||||
if command == "process" && len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
|
||||||
front, parseArgs = args[0], args[1:]
|
|
||||||
}
|
|
||||||
if err := fs.Parse(parseArgs); err != nil {
|
|
||||||
if errors.Is(err, flag.ErrHelp) {
|
|
||||||
return configInput{}, true, nil
|
|
||||||
}
|
|
||||||
return configInput{}, false, err
|
|
||||||
}
|
|
||||||
positions := fs.Args()
|
|
||||||
if front != "" {
|
|
||||||
positions = append([]string{front}, positions...)
|
|
||||||
}
|
|
||||||
if command == "process" {
|
|
||||||
if len(positions) != 1 {
|
|
||||||
return configInput{}, false, fmt.Errorf("expected exactly one transcript JSON path")
|
|
||||||
}
|
|
||||||
input.TranscriptPath = positions[0]
|
|
||||||
input.GlossaryPath = fs.Lookup("glossary").Value.String()
|
|
||||||
input.OutputPath = fs.Lookup("output").Value.String()
|
|
||||||
if strings.TrimSpace(input.GlossaryPath) == "" {
|
|
||||||
return configInput{}, false, fmt.Errorf("--glossary is required")
|
|
||||||
}
|
|
||||||
} else if len(positions) != 0 {
|
|
||||||
return configInput{}, false, fmt.Errorf("unexpected positional arguments")
|
|
||||||
}
|
|
||||||
input.ConfigPath, input.Overlay = *configPath, *overlay
|
|
||||||
if command == "print-effective" {
|
|
||||||
input.View = fs.Lookup("view").Value.String()
|
|
||||||
switch input.View {
|
|
||||||
case "settings", "profiles", "explain":
|
|
||||||
default:
|
|
||||||
return configInput{}, false, fmt.Errorf("unknown effective view %q", input.View)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs.Visit(func(f *flag.Flag) {
|
|
||||||
if f.Name == "config" {
|
|
||||||
input.ConfigPathSet = true
|
|
||||||
}
|
|
||||||
path, ok := settingPaths[f.Name]
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var value any = f.Value.String()
|
|
||||||
switch f.Name {
|
|
||||||
case "reporting", "debug":
|
|
||||||
value, _ = strconv.ParseBool(f.Value.String())
|
|
||||||
case "modules":
|
|
||||||
modules, err := config.ParseModulesCSV(f.Value.String())
|
|
||||||
if err != nil {
|
|
||||||
value = err
|
|
||||||
} else {
|
|
||||||
values := make([]any, len(modules))
|
|
||||||
for i, module := range modules {
|
|
||||||
values[i] = module
|
|
||||||
}
|
|
||||||
value = values
|
|
||||||
}
|
|
||||||
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
|
|
||||||
value, _ = strconv.Atoi(f.Value.String())
|
|
||||||
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
|
|
||||||
value, _ = strconv.ParseFloat(f.Value.String(), 64)
|
|
||||||
}
|
|
||||||
input.Overrides = append(input.Overrides, ck.Override{Path: path, Value: value, Flag: "--" + f.Name})
|
|
||||||
})
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
if err, ok := override.Value.(error); ok {
|
|
||||||
return configInput{}, false, fmt.Errorf("%s: %w", override.Flag, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return input, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectConfigPath(explicit string, set bool, stat func(string) (os.FileInfo, error), candidates []string) (string, string, error) {
|
|
||||||
if set {
|
|
||||||
if strings.TrimSpace(explicit) == "" {
|
|
||||||
return "", "", fmt.Errorf("--config requires a nonempty path")
|
|
||||||
}
|
|
||||||
if _, err := stat(explicit); err != nil {
|
|
||||||
return "", "", fmt.Errorf("config file %q: %w", explicit, err)
|
|
||||||
}
|
|
||||||
return explicit, "flag", nil
|
|
||||||
}
|
|
||||||
for _, path := range candidates {
|
|
||||||
if _, err := stat(path); err == nil {
|
|
||||||
return path, "default", nil
|
|
||||||
} else if !os.IsNotExist(err) {
|
|
||||||
return "", "", fmt.Errorf("config file %q: %w", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", "", fmt.Errorf("no configuration file found; select a config with backend and profile settings")
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadConfigInput(input configInput, command string) (selectedConfig, error) {
|
|
||||||
if command == "validate" && (!input.ConfigPathSet || strings.TrimSpace(input.ConfigPath) == "") {
|
|
||||||
return selectedConfig{}, fmt.Errorf("--config is required")
|
|
||||||
}
|
|
||||||
path, source, err := selectConfigPath(input.ConfigPath, input.ConfigPathSet, os.Stat, systemConfigPaths)
|
|
||||||
if err != nil {
|
|
||||||
return selectedConfig{}, err
|
|
||||||
}
|
|
||||||
loaded, err := config.Load(ck.File(path), input.Overlay, input.Overrides...)
|
|
||||||
if err != nil {
|
|
||||||
return selectedConfig{}, err
|
|
||||||
}
|
|
||||||
return selectedConfig{Path: path, Source: source, Result: loaded}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func configInputError(err error) string {
|
|
||||||
var loadErr *ck.LoadError
|
|
||||||
if !errors.As(err, &loadErr) {
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
var messages []string
|
|
||||||
for _, diagnostic := range loadErr.Diagnostics() {
|
|
||||||
messages = append(messages, diagnostic.Message)
|
|
||||||
}
|
|
||||||
if len(messages) == 0 {
|
|
||||||
return err.Error()
|
|
||||||
}
|
|
||||||
return strings.Join(messages, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func selectedReportDir(cfg config.Config, outputPath string) (string, error) {
|
|
||||||
if !cfg.Reporting.Enabled {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
if outputPath != "" {
|
|
||||||
return filepath.Dir(outputPath), nil
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(cfg.Reporting.StdoutDir) == "" {
|
|
||||||
return "", fmt.Errorf("report directory is required when reporting is enabled and transcript output is stdout")
|
|
||||||
}
|
|
||||||
return cfg.Reporting.StdoutDir, nil
|
|
||||||
}
|
|
||||||
@@ -1,250 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestSharedConfigInput(t *testing.T) {
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
t.Run(command, func(t *testing.T) {
|
|
||||||
args := []string{"--config", "main.yml", "--overlay", "testing", "--proposal-profile", "economy", "--modules", "", "--target-sections", "0"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, help, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil || help {
|
|
||||||
t.Fatalf("parse: %+v %v %v", input, help, err)
|
|
||||||
}
|
|
||||||
if !input.ConfigPathSet || input.ConfigPath != "main.yml" || input.Overlay != "testing" || len(input.Overrides) != 3 {
|
|
||||||
t.Fatalf("selection: %+v", input)
|
|
||||||
}
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
switch override.Path.String() {
|
|
||||||
case ck.Keys("pipeline", "modules").String():
|
|
||||||
if values, ok := override.Value.([]any); !ok || len(values) != 0 {
|
|
||||||
t.Fatalf("empty modules: %#v", override.Value)
|
|
||||||
}
|
|
||||||
case ck.Keys("chunking", "target_sections").String():
|
|
||||||
if override.Value != 0 {
|
|
||||||
t.Fatalf("zero target: %#v", override.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigInputRejectsRemovedAndProcessOnlyFlags(t *testing.T) {
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
for _, flag := range []string{"--model", "--llm-api-key", "--max-retries", "--llm-concurrency", "--report-json", "--work-dir-retention"} {
|
|
||||||
args := []string{flag, "value"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput(command, args, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted %s", command, flag)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, command := range []string{"validate", "print-effective"} {
|
|
||||||
if _, _, err := parseConfigInput(command, []string{"--glossary", "glossary.yml"}, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted process path", command)
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput(command, []string{"transcript.json"}, io.Discard); err == nil {
|
|
||||||
t.Fatalf("%s accepted positional", command)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
_, help, err := parseConfigInput(command, []string{"--help"}, io.Discard)
|
|
||||||
if err != nil || !help {
|
|
||||||
t.Fatalf("%s help: %v %v", command, help, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigSelection(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
|
|
||||||
if err := os.WriteFile(second, []byte("version: 2\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path, source, err := selectConfigPath("", false, os.Stat, []string{first, second})
|
|
||||||
if err != nil || path != second || source != "default" {
|
|
||||||
t.Fatalf("discovery: %q %q %v", path, source, err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
explicit string
|
|
||||||
set bool
|
|
||||||
candidates []string
|
|
||||||
}{
|
|
||||||
{"", true, []string{second}}, {first, true, []string{second}}, {"", false, []string{first}},
|
|
||||||
} {
|
|
||||||
if _, _, err := selectConfigPath(tc.explicit, tc.set, os.Stat, tc.candidates); err == nil {
|
|
||||||
t.Fatalf("accepted unavailable selection: %+v", tc)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
statDenied := func(string) (os.FileInfo, error) { return nil, os.ErrPermission }
|
|
||||||
if _, _, err := selectConfigPath("", false, statDenied, []string{first}); !errors.Is(err, os.ErrPermission) {
|
|
||||||
t.Fatalf("inaccessible candidate: %v", err)
|
|
||||||
}
|
|
||||||
path, source, err = selectConfigPath(second, true, os.Stat, []string{first})
|
|
||||||
if err != nil || path != second || source != "flag" {
|
|
||||||
t.Fatalf("explicit path: %q %q %v", path, source, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) {
|
|
||||||
t.Setenv("AUDITA_CONFIG", filepath.Join(t.TempDir(), "legacy.yml"))
|
|
||||||
_, _, err := selectConfigPath("", false, os.Stat, []string{filepath.Join(t.TempDir(), "missing.yml")})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "no configuration file") {
|
|
||||||
t.Fatalf("legacy environment changed discovery: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEffectiveViewParsing(t *testing.T) {
|
|
||||||
for _, view := range []string{"settings", "profiles", "explain"} {
|
|
||||||
input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard)
|
|
||||||
if err != nil || input.View != view {
|
|
||||||
t.Fatalf("%q: %+v %v", view, input, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil {
|
|
||||||
t.Fatal("unknown view accepted")
|
|
||||||
}
|
|
||||||
if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil {
|
|
||||||
t.Fatal("validate accepted inspection-only flag")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
main := filepath.Join(dir, "config.yml")
|
|
||||||
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\nchunking:\n max_section_tokens: 900\n"
|
|
||||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
args := []string{"--config", main, "--overlay", "testing", "--min-section-tokens", "500"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, command)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
c := selected.Result.Config
|
|
||||||
if c.LLM.ProposalProfile != "testing" || c.LLM.ValidationProfile != "testing" || c.Chunking.MinSectionTokens != 500 || c.Chunking.MaxSectionTokens != 900 {
|
|
||||||
t.Fatalf("%s resolved %+v", command, c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput("validate", []string{"--overlay", "testing"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := loadConfigInput(input, "validate"); err == nil {
|
|
||||||
t.Fatal("validate accepted discovery")
|
|
||||||
}
|
|
||||||
input, _, err = parseConfigInput("validate", []string{"--config", main}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := loadConfigInput(input, "validate"); err == nil {
|
|
||||||
t.Fatal("unmodified invalid lower-precedence section relation accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
main := filepath.Join(dir, "config.yml")
|
|
||||||
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\n"
|
|
||||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("reporting:\n enabled: true\n stdout_dir: reports\ndiagnostics:\n debug: true\n"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defaultInput, _, err := parseConfigInput("validate", []string{"--config", main}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defaults, err := loadConfigInput(defaultInput, "validate")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !defaults.Result.Config.Reporting.Enabled || defaults.Result.Config.Reporting.StdoutDir != "." || defaults.Result.Config.Diagnostics.Debug {
|
|
||||||
t.Fatalf("unexpected defaults: %+v", defaults.Result.Config)
|
|
||||||
}
|
|
||||||
for _, command := range []string{"process", "validate", "print-effective"} {
|
|
||||||
args := []string{"--config", main, "--overlay", "testing", "--reporting=false", "--debug=false"}
|
|
||||||
if command == "process" {
|
|
||||||
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
|
|
||||||
}
|
|
||||||
input, _, err := parseConfigInput(command, args, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(input.Overrides) != 2 || input.Overrides[0].Value != false || input.Overrides[1].Value != false {
|
|
||||||
t.Fatalf("boolean overrides must remain typed: %+v", input.Overrides)
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, command)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if selected.Result.Config.Reporting.Enabled || selected.Result.Config.Diagnostics.Debug || selected.Result.Config.Reporting.StdoutDir != "reports" {
|
|
||||||
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
|
|
||||||
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
|
|
||||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "reports" {
|
|
||||||
t.Fatalf("stdout destination: %q %v", got, err)
|
|
||||||
}
|
|
||||||
cfg.Reporting.StdoutDir = ""
|
|
||||||
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json")); err != nil || got != "out" {
|
|
||||||
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
|
|
||||||
}
|
|
||||||
if _, err := selectedReportDir(cfg, ""); err == nil {
|
|
||||||
t.Fatal("enabled stdout reporting accepted blank destination")
|
|
||||||
}
|
|
||||||
cfg.Reporting.Enabled = false
|
|
||||||
if got, err := selectedReportDir(cfg, ""); err != nil || got != "" {
|
|
||||||
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
|
|
||||||
input, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--reporting", "--debug", "--report-dir", "reports"}, io.Discard)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
values := map[string]any{}
|
|
||||||
for _, override := range input.Overrides {
|
|
||||||
values[override.Path.String()] = override.Value
|
|
||||||
}
|
|
||||||
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
|
|
||||||
t.Fatalf("incorrect visited overrides: %+v", values)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
150
internal/cli/examples_integration_test.go
Normal file
150
internal/cli/examples_integration_test.go
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
|
||||||
|
_, file, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("cannot locate examples")
|
||||||
|
}
|
||||||
|
root := filepath.Join(filepath.Dir(file), "..", "..", "examples")
|
||||||
|
t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key")
|
||||||
|
for _, tc := range []struct{ dir, main, overlay string }{
|
||||||
|
{"local", "config.yml", ""}, {"service", "config.yml", "testing"},
|
||||||
|
{"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""},
|
||||||
|
} {
|
||||||
|
t.Run(tc.dir+"/"+tc.main, func(t *testing.T) {
|
||||||
|
main := filepath.Join(root, tc.dir, tc.main)
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded.Config.Reporting.Enabled = false
|
||||||
|
var calls atomic.Int32
|
||||||
|
var mu sync.Mutex
|
||||||
|
var models, prompts []string
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var content strings.Builder
|
||||||
|
for _, message := range request.Messages {
|
||||||
|
content.WriteString(message.Content)
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
models = append(models, request.Model)
|
||||||
|
prompts = append(prompts, content.String())
|
||||||
|
mu.Unlock()
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil
|
||||||
|
})
|
||||||
|
engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source}
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
if _, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
|
||||||
|
t.Fatalf("provider calls=%d", calls.Load())
|
||||||
|
}
|
||||||
|
if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") {
|
||||||
|
t.Fatalf("overlay models=%v", models)
|
||||||
|
}
|
||||||
|
if tc.dir == "campaigns" && tc.main == "first.yml" {
|
||||||
|
if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) {
|
||||||
|
t.Fatalf("campaign glossary selections: %v", prompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.dir == "campaigns" && tc.main == "second.yml" {
|
||||||
|
if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) {
|
||||||
|
t.Fatalf("second campaign glossary selection: %v", prompts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) {
|
||||||
|
_, file, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("cannot locate examples")
|
||||||
|
}
|
||||||
|
source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit")
|
||||||
|
dir := t.TempDir()
|
||||||
|
for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(source, name))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inspect := func() string {
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
engine, err := llm.NewRuntime(context.Background(), loaded, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return prompt.Messages[0].Content
|
||||||
|
}
|
||||||
|
before := inspect()
|
||||||
|
filePath := filepath.Join(dir, "prompts.yml")
|
||||||
|
raw, err := os.ReadFile(filePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after := inspect()
|
||||||
|
if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") {
|
||||||
|
t.Fatalf("before=%q after=%q", before, after)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,442 +1,110 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/normalization"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/outputschema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/reporting"
|
||||||
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/preflight"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
"go.maximumdirect.net/ckconfig"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
ck "go.maximumdirect.net/configkit"
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) int {
|
// processResult carries the runner's transcript and ordinary report facts.
|
||||||
input, help, err := parseConfigInput("process", args, stderr)
|
type processResult struct {
|
||||||
if err != nil {
|
Transcript *schema.Transcript
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
|
Snapshot reporting.Snapshot
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if help {
|
|
||||||
writeProcessUsage(stdout)
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
selected, err := loadConfigInput(input, "process")
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
cfg := selected.Result.Config
|
|
||||||
reportDir, err := selectedReportDir(cfg, input.OutputPath)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if cfg.Diagnostics.Debug && strings.TrimSpace(cfg.Diagnostics.WorkDir) == "" {
|
|
||||||
fmt.Fprintln(stderr, "audita process: debug work directory is required")
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", err)
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
runtime, err := llm.NewRuntime(ctx, cfg, selected.Path, nil)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
if ctx.Err() != nil || llm.IsAssetError(err) {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
specs, err := contracts.ResolveModuleRunSpecs(cfg.Pipeline.Modules)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
runID, err := generateRunID()
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: run ID: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
started := time.Now().UTC()
|
|
||||||
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
|
|
||||||
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
|
|
||||||
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
|
|
||||||
protected := []string{input.GlossaryPath}
|
|
||||||
for _, source := range selected.Result.Sources() {
|
|
||||||
if source.Path != "" {
|
|
||||||
protected = append(protected, source.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
protected = append(protected, runtime.ProfilePaths...)
|
|
||||||
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
|
|
||||||
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 2
|
|
||||||
}
|
|
||||||
state.session = session
|
|
||||||
initial := state.snapshot(reporting.StatusInProgress, nil, nil)
|
|
||||||
initialBundle, err := state.bundle(initial)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
initFailure := []processreport.RunFailure{{Phase: "initialization", Reason: outcomes.ReasonInternalFailure}}
|
|
||||||
failedInit, err := state.bundle(state.snapshot(reporting.StatusFailed, nil, initFailure))
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if err := session.Initialize(initialBundle, failedInit); err != nil {
|
|
||||||
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
state.startDebug()
|
|
||||||
return state.execute()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type processState struct {
|
func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcome {
|
||||||
cfg config.Config
|
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
|
||||||
runtime *llm.Runtime
|
|
||||||
specs []contracts.ModuleRunSpec
|
|
||||||
runID string
|
|
||||||
started time.Time
|
|
||||||
transcriptPath string
|
|
||||||
glossaryPath string
|
|
||||||
outputPath string
|
|
||||||
selected selectedConfig
|
|
||||||
overlay string
|
|
||||||
ctx context.Context
|
|
||||||
stdout io.Writer
|
|
||||||
stderr io.Writer
|
|
||||||
session *diagnostics.OutputSession
|
|
||||||
debugDir *diagnostics.RunDirectory
|
|
||||||
debugWarnings []string
|
|
||||||
debugMu sync.Mutex
|
|
||||||
runOutput *runner.RunOutput
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateRunID() (string, error) {
|
|
||||||
var random [12]byte
|
|
||||||
if _, err := rand.Read(random[:]); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return "run-" + hex.EncodeToString(random[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
|
|
||||||
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
|
|
||||||
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
|
||||||
OutputPath: s.outputPath, ModuleSpecs: s.specs, RunOutput: s.runOutput, RunFailures: failures})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) bundle(snapshot reporting.Snapshot) (diagnostics.ReportBundle, error) {
|
|
||||||
if !s.cfg.Reporting.Enabled {
|
|
||||||
return diagnostics.ReportBundle{}, nil
|
|
||||||
}
|
|
||||||
secrets := llm.ConfiguredSecrets(&s.cfg)
|
|
||||||
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument(), secrets...)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diagnostics.ReportBundle{}, err
|
if canceled := inv.Context.Err(); canceled != nil && !errors.Is(err, canceled) {
|
||||||
|
err = errors.Join(err, canceled)
|
||||||
}
|
}
|
||||||
recoveriesYAML, err := reporting.YAML(snapshot.RecoveriesDocument(), secrets...)
|
var loadErr *ck.LoadError
|
||||||
|
if errors.As(err, &loadErr) {
|
||||||
|
_, explanation := ckconfig.AdaptDiagnostics("Application configuration:\n", loadErr.Sources(), loadErr.Diagnostics())
|
||||||
|
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
|
||||||
|
return commandkit.Usage(loadErr.Error(), explanation.Details...).WithCause(err)
|
||||||
|
}
|
||||||
|
return commandkit.Failure(loadErr.Error(), explanation.Details...).WithCause(err)
|
||||||
|
}
|
||||||
|
if inv.Context.Err() != nil || llm.IsAssetError(err) {
|
||||||
|
return commandkit.Failure("Could not prepare processing.").WithCause(err)
|
||||||
|
}
|
||||||
|
return commandkit.Usage(err.Error()).WithCause(err)
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return diagnostics.ReportBundle{}, err
|
var selection *preflight.SelectionError
|
||||||
|
if inv.Context.Err() == nil && errors.As(err, &selection) {
|
||||||
|
return commandkit.Usage(err.Error()).WithCause(err)
|
||||||
}
|
}
|
||||||
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML,
|
return commandkit.Failure(err.Error()).WithCause(err)
|
||||||
Markdown: reporting.Markdown(snapshot, secrets...)}, nil
|
|
||||||
}
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
func (s *processState) debugWrite(label string, write func(*diagnostics.RunDirectory) error) {
|
var warningsMu sync.Mutex
|
||||||
if s.debugDir == nil {
|
warn := func(err error) {
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := write(s.debugDir); err != nil {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("%s: %w", label, err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) startDebug() {
|
|
||||||
if !s.cfg.Diagnostics.Debug {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
dir, err := diagnostics.NewRunDirectory(s.cfg.Diagnostics.WorkDir, s.runID, llm.ConfiguredSecrets(&s.cfg)...)
|
|
||||||
if err != nil {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("directory: %w", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.debugDir = dir
|
|
||||||
s.debugWrite("invocation", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteInvocationMetadata(diagnostics.InvocationMetadata{
|
|
||||||
Operation: "process", TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
|
|
||||||
OutputPath: s.outputPath, ConfigPath: s.selected.Path, ConfigSource: s.selected.Source,
|
|
||||||
Overlay: s.overlay, ConfigVersion: &s.cfg.Version, TranscriptDescription: s.cfg.Context.Description,
|
|
||||||
Modules: append([]string(nil), s.cfg.Pipeline.Modules...), RunID: s.runID, StartedAt: s.started,
|
|
||||||
})
|
|
||||||
})
|
|
||||||
var effective bytes.Buffer
|
|
||||||
if err := s.selected.Result.WriteEffective(&effective); err == nil {
|
|
||||||
s.debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) })
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("effective settings: %w", err))
|
|
||||||
}
|
|
||||||
if origins, err := llm.ExplanationYAML(s.selected.Result); err == nil {
|
|
||||||
s.debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("configuration origins: %w", err))
|
|
||||||
}
|
|
||||||
if profiles, err := s.runtime.SelectedProfilesYAML(); err == nil {
|
|
||||||
s.debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
s.reportDiagnosticError(fmt.Errorf("selected profiles: %w", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) debugPath() string {
|
|
||||||
if s.debugDir == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return s.debugDir.Path()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) execute() int {
|
|
||||||
transcriptBytes, err := coreio.ReadRequiredFile(s.transcriptPath, "transcript")
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
glossaryBytes, err := coreio.ReadRequiredFile(s.glossaryPath, "glossary")
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
source, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInputInvalid, err)
|
|
||||||
}
|
|
||||||
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("input", outcomes.ReasonInputInvalid, err)
|
|
||||||
}
|
|
||||||
s.debugWrite("source transcript", func(d *diagnostics.RunDirectory) error { return d.WriteSourceTranscript(source, transcriptBytes) })
|
|
||||||
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
|
||||||
MaxSegmentGap: s.cfg.Normalization.MaxSegmentGap, EllipsisGap: s.cfg.Normalization.EllipsisGap,
|
|
||||||
MaxSegmentDuration: s.cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: s.cfg.Normalization.MaxSegmentTokens,
|
|
||||||
})
|
|
||||||
normalized, normSummary := normalizer.Normalize(sourceToCanonicalTranscript(source))
|
|
||||||
s.debugWrite("normalized transcript", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizedTranscript(normalized) })
|
|
||||||
s.debugWrite("normalization summary", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizationSummary(normSummary) })
|
|
||||||
if err := s.ctx.Err(); err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonCanceled, err)
|
|
||||||
}
|
|
||||||
observations := llm.NewObservationCollector()
|
|
||||||
proposalClient, err := s.runtime.NewRoleClient("proposal", observations.Hooks())
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
validationClient, err := s.runtime.NewRoleClient("validation", observations.Hooks())
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("processing", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
factory := modules.NewFactory(modules.Dependencies{Config: &s.cfg, Glossary: glossary, DiagnosticsDir: s.debugPath()})
|
|
||||||
result, runErr := runner.New(factory).Run(s.ctx, runner.RunInput{
|
|
||||||
Config: &s.cfg, Transcript: normalized, Glossary: glossary, ModuleSpecs: s.specs,
|
|
||||||
ProposalLLMClient: proposalClient, ProposalDiagnosticsDir: s.debugPath(),
|
|
||||||
ValidationLLMClient: validationClient, ValidationDiagnosticsDir: s.debugPath(),
|
|
||||||
CaptureCandidates: s.debugDir != nil,
|
|
||||||
ReportDiagnosticError: s.reportDiagnosticError,
|
|
||||||
})
|
|
||||||
result.Observations = observations.Snapshot()
|
|
||||||
result.ObservationsAvailable = true
|
|
||||||
result.LLMSummary = llm.Summarize(result.Observations)
|
|
||||||
s.runOutput = &result
|
|
||||||
s.writeExecutionEvidence()
|
|
||||||
if len(result.Sections) > 0 || runErr == nil {
|
|
||||||
s.writeChunkSummary(result.Sections)
|
|
||||||
}
|
|
||||||
if runErr != nil {
|
|
||||||
return s.fail("processing", fatalReason(s.ctx, runErr), runErr)
|
|
||||||
}
|
|
||||||
encoder, err := outputschema.Resolve(s.cfg.Output.Schema)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("encoding", outcomes.ReasonSetupFailure, err)
|
|
||||||
}
|
|
||||||
outputBytes, err := encoder.Encoder(result.FinalTranscript)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("encoding", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
return s.finish(outputBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) writeChunkSummary(inventory []contracts.SectionMetadata) {
|
|
||||||
if s.debugDir == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sections := make([]chunking.Section, len(inventory))
|
|
||||||
for i, metadata := range inventory {
|
|
||||||
sections[i] = chunking.Section{Index: metadata.Index, StartSegmentID: metadata.StartSegmentID,
|
|
||||||
EndSegmentID: metadata.EndSegmentID, EstimatedTokens: metadata.EstimatedTokens,
|
|
||||||
Segments: make([]schema.Segment, metadata.SegmentCount)}
|
|
||||||
}
|
|
||||||
config := chunking.ChunkingConfig{TargetSections: s.cfg.Chunking.TargetSections,
|
|
||||||
MaxSectionTokens: s.cfg.Chunking.MaxSectionTokens, MinSectionTokens: s.cfg.Chunking.MinSectionTokens}
|
|
||||||
summary := chunking.ComputeDetailedSummary(sections, config)
|
|
||||||
s.debugWrite("chunking summary", func(d *diagnostics.RunDirectory) error { return d.WriteChunkingSummary(&summary) })
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) writeExecutionEvidence() {
|
|
||||||
if s.debugDir == nil || s.runOutput == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
output := s.runOutput
|
|
||||||
s.debugWrite("model execution", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations))
|
|
||||||
})
|
|
||||||
s.debugWrite("utilization", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, s.cfg.Backends))
|
|
||||||
})
|
|
||||||
s.debugWrite("correction ledger", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
|
|
||||||
RunDirectoryPath: d.Path(), RunOutput: output}))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func fatalReason(ctx context.Context, err error) outcomes.ReasonCode {
|
|
||||||
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
|
|
||||||
return outcomes.ReasonCanceled
|
|
||||||
}
|
|
||||||
var failure *structuredoutput.Failure
|
|
||||||
if errors.As(err, &failure) && failure.Reason != "" {
|
|
||||||
return failure.Reason
|
|
||||||
}
|
|
||||||
return outcomes.ReasonInternalFailure
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) fail(phase string, reason outcomes.ReasonCode, cause error) int {
|
|
||||||
s.debugWrite("error log", func(d *diagnostics.RunDirectory) error {
|
|
||||||
return d.WriteErrorLog(phase + ": " + llm.RedactError(cause, s.cfg))
|
|
||||||
})
|
|
||||||
failures := []processreport.RunFailure{{Phase: phase, Reason: reason}}
|
|
||||||
completed := time.Now().UTC()
|
|
||||||
base, err := s.bundle(s.snapshot(reporting.StatusFailed, &completed, failures))
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(err, s.cfg))
|
|
||||||
}
|
|
||||||
writeFailures := s.session.Finalize(nil, base, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
|
|
||||||
all := append(append([]processreport.RunFailure(nil), failures...), mapWriteFailures(extra)...)
|
|
||||||
bundle, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, all))
|
|
||||||
if renderErr != nil {
|
|
||||||
return base
|
|
||||||
}
|
|
||||||
return bundle
|
|
||||||
})
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(cause, s.cfg))
|
|
||||||
s.printFailures(writeFailures)
|
|
||||||
s.printDebugWarnings()
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) finish(outputBytes []byte) int {
|
|
||||||
completed := time.Now().UTC()
|
|
||||||
snapshot := s.snapshot(reporting.StatusCompleted, &completed, nil)
|
|
||||||
complete, err := s.bundle(snapshot)
|
|
||||||
if err != nil {
|
|
||||||
return s.fail("report_write", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
if s.session.TranscriptPath() == "" {
|
|
||||||
if _, err := s.stdout.Write(outputBytes); err != nil {
|
|
||||||
return s.fail("transcript_write", outcomes.ReasonInternalFailure, err)
|
|
||||||
}
|
|
||||||
outputBytes = nil
|
|
||||||
}
|
|
||||||
writeFailures := s.session.Finalize(outputBytes, complete, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
|
|
||||||
failed, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, mapWriteFailures(extra)))
|
|
||||||
if renderErr != nil {
|
|
||||||
return diagnostics.ReportBundle{}
|
|
||||||
}
|
|
||||||
return failed
|
|
||||||
})
|
|
||||||
if len(writeFailures) > 0 {
|
|
||||||
s.printFailures(writeFailures)
|
|
||||||
s.printDebugWarnings()
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
fmt.Fprintln(s.stderr, reporting.CompletionLine(snapshot))
|
|
||||||
for _, path := range s.session.ReportPaths() {
|
|
||||||
fmt.Fprintf(s.stderr, "Report: %s\n", s.safePath(path))
|
|
||||||
}
|
|
||||||
if s.debugDir != nil {
|
|
||||||
fmt.Fprintf(s.stderr, "Debug: %s\n", s.safePath(s.debugDir.Path()))
|
|
||||||
}
|
|
||||||
s.printDebugWarnings()
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func mapWriteFailures(failures []diagnostics.FileFailure) []processreport.RunFailure {
|
|
||||||
out := make([]processreport.RunFailure, len(failures))
|
|
||||||
for i, failure := range failures {
|
|
||||||
phase := "report_write"
|
|
||||||
if failure.Kind == "transcript write" {
|
|
||||||
phase = "transcript_write"
|
|
||||||
}
|
|
||||||
out[i] = processreport.RunFailure{Phase: phase, Reason: outcomes.ReasonInternalFailure, OutputPath: failure.Path}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) printFailures(failures []diagnostics.FileFailure) {
|
|
||||||
for _, failure := range failures {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(failure, s.cfg))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) printDebugWarnings() {
|
|
||||||
s.debugMu.Lock()
|
|
||||||
defer s.debugMu.Unlock()
|
|
||||||
for _, warning := range s.debugWarnings {
|
|
||||||
fmt.Fprintf(s.stderr, "audita process: %s\n", warning)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// safePath sanitizes presentation only; filesystem destinations remain literal.
|
|
||||||
func (s *processState) safePath(path string) string {
|
|
||||||
return redaction.New(llm.ConfiguredSecrets(&s.cfg)).String(path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *processState) reportDiagnosticError(err error) {
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
message := "debug " + llm.RedactError(err, s.cfg)
|
warningsMu.Lock()
|
||||||
s.debugMu.Lock()
|
prepared.Prepared.Warnings = append(prepared.Prepared.Warnings, err)
|
||||||
defer s.debugMu.Unlock()
|
warningsMu.Unlock()
|
||||||
s.debugWarnings = append(s.debugWarnings, message)
|
}
|
||||||
|
debug := prepared.Prepared.Debug
|
||||||
|
if debug != nil {
|
||||||
|
warn(debug.WriteSourceTranscript(loaded.Source, loaded.RawTranscript))
|
||||||
|
}
|
||||||
|
loaded.RawTranscript = nil
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
|
||||||
|
MaxSegmentGap: cfg.Normalization.MaxSegmentGap, EllipsisGap: cfg.Normalization.EllipsisGap,
|
||||||
|
MaxSegmentDuration: cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: cfg.Normalization.MaxSegmentTokens,
|
||||||
|
})
|
||||||
|
transcript := loaded.Source.Canonical()
|
||||||
|
loaded.Source = nil
|
||||||
|
normalized, summary := normalizer.Normalize(transcript)
|
||||||
|
if debug != nil {
|
||||||
|
warn(debug.WriteNormalizedTranscript(normalized))
|
||||||
|
warn(debug.WriteNormalizationSummary(summary))
|
||||||
|
}
|
||||||
|
if err := inv.Context.Err(); err != nil {
|
||||||
|
return finishProcess(inv, input, prepared, reporting.Snapshot{}, nil, &runner.ExecutionFailure{Phase: "normalization", Cause: err})
|
||||||
|
}
|
||||||
|
reporter := inv.Reporter()
|
||||||
|
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn, func(message string) {
|
||||||
|
_ = reporter.Progress(message)
|
||||||
|
})
|
||||||
|
if runErr != nil {
|
||||||
|
return finishProcess(inv, input, prepared, result.Snapshot, nil, runErr)
|
||||||
|
}
|
||||||
|
encoder, err := outputschema.Resolve(cfg.Output.Schema)
|
||||||
|
if err != nil {
|
||||||
|
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: err})
|
||||||
|
}
|
||||||
|
if result.Transcript == nil {
|
||||||
|
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: fmt.Errorf("processing returned no transcript")})
|
||||||
|
}
|
||||||
|
payload, err := encoder.Encoder(result.Transcript)
|
||||||
|
if err != nil {
|
||||||
|
return finishProcess(inv, input, prepared, result.Snapshot, nil, &runner.ExecutionFailure{Phase: "encoding", Cause: err})
|
||||||
|
}
|
||||||
|
return finishProcess(inv, input, prepared, result.Snapshot, payload, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugPath returns the retained artifact directory, if its preparation succeeded.
|
||||||
|
func (p *preparedProcess) DebugPath() string {
|
||||||
|
if p == nil || p.Prepared == nil || p.Prepared.Debug == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return p.Prepared.Debug.Path()
|
||||||
}
|
}
|
||||||
|
|||||||
118
internal/cli/process_command.go
Normal file
118
internal/cli/process_command.go
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processInput struct {
|
||||||
|
Config commandkit.Supplied[string]
|
||||||
|
Overlay commandkit.Supplied[string]
|
||||||
|
Transcript string
|
||||||
|
Output commandkit.Supplied[string]
|
||||||
|
Debug commandkit.Supplied[bool]
|
||||||
|
Overrides []ck.Override
|
||||||
|
}
|
||||||
|
|
||||||
|
func processCommand() commandkit.Command {
|
||||||
|
stringValue := func(s string) (string, error) { return s, nil }
|
||||||
|
flags := []commandkit.Flag[processInput]{
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Application configuration file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Config = v }),
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Overlay = v }),
|
||||||
|
commandkit.ValueFlag(commandkit.FlagMeta{Name: "output", Usage: "Transcript output file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Output = v }),
|
||||||
|
commandkit.BoolFlag(commandkit.FlagMeta{Name: "debug", Usage: "Capture diagnostic artifacts."}, false, func(in *processInput, v commandkit.Supplied[bool]) { in.Debug = v }),
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"proposal-profile", ck.Keys("llm", "proposal_profile")}, {"validation-profile", ck.Keys("llm", "validation_profile")},
|
||||||
|
{"output-schema", ck.Keys("output", "schema")}, {"transcript-description", ck.Keys("context", "description")},
|
||||||
|
{"report-dir", ck.Keys("reporting", "stdout_dir")}, {"work-dir", ck.Keys("diagnostics", "work_dir")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*string)(nil), stringValue,
|
||||||
|
func(in *processInput, v commandkit.Supplied[string]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"validation-max-prompt-tokens", ck.Keys("validation", "max_prompt_tokens")}, {"target-sections", ck.Keys("chunking", "target_sections")},
|
||||||
|
{"max-section-tokens", ck.Keys("chunking", "max_section_tokens")}, {"min-section-tokens", ck.Keys("chunking", "min_section_tokens")},
|
||||||
|
{"normalize-max-segment-tokens", ck.Keys("normalization", "max_segment_tokens")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*int)(nil), strconv.Atoi,
|
||||||
|
func(in *processInput, v commandkit.Supplied[int]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"normalize-max-segment-gap", ck.Keys("normalization", "max_segment_gap")}, {"normalize-ellipsis-gap", ck.Keys("normalization", "ellipsis_gap")},
|
||||||
|
{"normalize-max-segment-duration", ck.Keys("normalization", "max_segment_duration")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*float64)(nil), func(s string) (float64, error) { return strconv.ParseFloat(s, 64) },
|
||||||
|
func(in *processInput, v commandkit.Supplied[float64]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
path ck.Path
|
||||||
|
}{
|
||||||
|
{"glossaries", ck.Keys("glossaries")}, {"protected-terms", ck.Keys("protected_terms")},
|
||||||
|
} {
|
||||||
|
name, path := item.name, item.path
|
||||||
|
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "JSON array of collection names, or null."}, (*any)(nil), parseCollectionNames,
|
||||||
|
func(in *processInput, value commandkit.Supplied[any]) {
|
||||||
|
if value.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: value.Value, Flag: "--" + name})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
flags = append(flags, commandkit.BoolFlag(commandkit.FlagMeta{Name: "reporting", Usage: "Enable ordinary reports."}, false,
|
||||||
|
func(in *processInput, v commandkit.Supplied[bool]) {
|
||||||
|
if v.Set {
|
||||||
|
in.Overrides = append(in.Overrides, ck.Override{Path: ck.Keys("reporting", "enabled"), Value: v.Value, Flag: "--reporting"})
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
return commandkit.NewCommand(commandkit.CommandSpec[processInput]{Name: "process", Summary: "Polish a transcript.", Flags: flags, Run: runProcess,
|
||||||
|
Arguments: []commandkit.Argument[processInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "transcript", Usage: "Transcript JSON file."}, true, stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Transcript = v.Value })},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseCollectionNames(raw string) (any, error) {
|
||||||
|
value, err := commandkit.ParseCLIJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if value == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
names, ok := value.([]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("expected a JSON array of collection names or null")
|
||||||
|
}
|
||||||
|
for _, name := range names {
|
||||||
|
if _, ok := name.(string); !ok {
|
||||||
|
return nil, fmt.Errorf("collection names must be strings")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
157
internal/cli/process_command_integration_test.go
Normal file
157
internal/cli/process_command_integration_test.go
Normal file
@@ -0,0 +1,157 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProcessShowsConfigurationLoadDiagnostics(t *testing.T) {
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, config, finding string
|
||||||
|
}{
|
||||||
|
{"obsolete version", "version: 2\n", "docs/migrations/v3.md"},
|
||||||
|
{"invalid retry budget", "version: 3\nllm: {transient_retries: -1}\n", "retry budget must be non-negative"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte(tc.config), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := filepath.Join(dir, "out.json")
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--quiet", "--config", configPath, "--output", output, filepath.Join(dir, "transcript.json")}
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
var loadErr *ck.LoadError
|
||||||
|
if result.Status != 2 || !errors.As(result.Cause, &loadErr) || out.Len() != 0 ||
|
||||||
|
!strings.Contains(stderr.String(), tc.finding) || !strings.Contains(stderr.String(), configPath) {
|
||||||
|
t.Fatalf("status=%d cause=%v out=%q err=%q", result.Status, result.Cause, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil || len(entries) != 1 {
|
||||||
|
t.Fatalf("outputs created on load failure: %v, err=%v", entries, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessUsesResolvedReportingMode(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"blocked": "file",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
status int
|
||||||
|
stdout bool
|
||||||
|
info bool
|
||||||
|
warning bool
|
||||||
|
}{
|
||||||
|
{"ordinary", nil, 0, true, false, false},
|
||||||
|
{"verbose", []string{"--verbose"}, 0, true, true, false},
|
||||||
|
{"quiet stdout", []string{"--quiet"}, 2, false, false, false},
|
||||||
|
{"quiet file", []string{"--quiet", "--output", filepath.Join(dir, "quiet.json")}, 0, false, false, false},
|
||||||
|
{"verbose after quiet", []string{"--quiet", "--verbose"}, 0, true, true, false},
|
||||||
|
{"quiet after verbose", []string{"--verbose", "--quiet", "--output", filepath.Join(dir, "ordered.json")}, 0, false, false, false},
|
||||||
|
{"debug disabled", []string{"--debug=false", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, false, false},
|
||||||
|
{"debug failure", []string{"--debug", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, true, true},
|
||||||
|
{"debug quiet", []string{"--debug", "--quiet", "--output", filepath.Join(dir, "debug-quiet.json"), "--work-dir", filepath.Join(dir, "blocked")}, 0, false, false, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--config", filepath.Join(dir, "config.yml")}
|
||||||
|
args = append(args, tc.flags...)
|
||||||
|
args = append(args, filepath.Join(dir, "transcript.json"))
|
||||||
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if result.Status != tc.status || (out.Len() > 0) != tc.stdout || strings.Contains(stderr.String(), "Completed:") != tc.info || strings.Contains(stderr.String(), "Debug artifact:") != tc.warning {
|
||||||
|
t.Fatalf("status=%d out=%q err=%q", result.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessChecksOnlyTheActiveReportDirectory(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
flags []string
|
||||||
|
wantReport bool
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
{"file output ignores blank configured directory", nil, true, false},
|
||||||
|
{"explicit blank directory", []string{"--report-dir="}, false, true},
|
||||||
|
{"disabled reporting ignores explicit blank directory", []string{"--reporting=false", "--report-dir="}, false, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
configPath := filepath.Join(dir, "config.yml")
|
||||||
|
if err := os.WriteFile(configPath, []byte("version: 3\npipeline: {stages: []}\nreporting: {stdout_dir: ''}\n"), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transcript := filepath.Join(dir, "transcript.json")
|
||||||
|
if err := os.WriteFile(transcript, []byte(`[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
output := filepath.Join(dir, "output.json")
|
||||||
|
args := append([]string{"process", "--config", configPath, "--output", output}, tc.flags...)
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
status := Run(context.Background(), append(args, transcript), &stdout, &stderr)
|
||||||
|
if (status != 0) != tc.wantError || (tc.wantError && !strings.Contains(stderr.String(), "report directory is required")) {
|
||||||
|
t.Fatalf("status=%d stdout=%q stderr=%q", status, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
_, err := os.Stat(filepath.Join(dir, "errors.yml"))
|
||||||
|
if (tc.wantReport && err != nil) || (!tc.wantReport && !os.IsNotExist(err)) {
|
||||||
|
t.Fatalf("report file: %v, want present=%t", err, tc.wantReport)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessCollectionFlags(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\nglossary_collections:\n party: {entries: []}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app, err := buildApplication()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
value string
|
||||||
|
status int
|
||||||
|
}{{`["party"]`, 0}, {`[]`, 0}, {`null`, 0}, {`[1]`, 2}, {`"party"`, 2}} {
|
||||||
|
var out, stderr bytes.Buffer
|
||||||
|
args := []string{"process", "--config", filepath.Join(dir, "config.yml"), "--glossaries", tc.value, filepath.Join(dir, "transcript.json")}
|
||||||
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
|
||||||
|
if outcome.Status != tc.status {
|
||||||
|
t.Fatalf("value=%q status=%d out=%q err=%q", tc.value, outcome.Status, out.String(), stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
179
internal/cli/process_inputs.go
Normal file
179
internal/cli/process_inputs.go
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/diagnostics"
|
||||||
|
coreio "gitea.maximumdirect.net/eric/audita/v4/internal/core/io"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/preflight"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/processreport"
|
||||||
|
"go.maximumdirect.net/ckconfig"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
pk "go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type processInputs struct {
|
||||||
|
Configuration *ck.Result[config.Config]
|
||||||
|
Runtime *llm.Runtime
|
||||||
|
Source *schema.SourceTranscript
|
||||||
|
RawTranscript []byte
|
||||||
|
ProtectedPaths []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadProcessInputs(ctx context.Context, input processInput, mode commandkit.ReportingMode) (*processInputs, error) {
|
||||||
|
if mode == commandkit.ReportingModeQuiet && input.Output.Value == "" {
|
||||||
|
return nil, fmt.Errorf("quiet processing requires an output file or /dev/null")
|
||||||
|
}
|
||||||
|
selection := ckconfig.Selection{Config: input.Config, Overlay: input.Overlay, Overrides: input.Overrides}
|
||||||
|
loaded, err := loadApplication(ctx, selection, struct{}{}, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runtime, err := llm.NewRuntime(ctx, loaded, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
raw, err := coreio.ReadRequiredFile(input.Transcript, "transcript")
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil, errors.Join(err, ctx.Err())
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !input.Debug.Value {
|
||||||
|
raw = nil
|
||||||
|
}
|
||||||
|
protected := append([]string(nil), runtime.SourcePaths...)
|
||||||
|
for _, entry := range loaded.Sources() {
|
||||||
|
if entry.Kind == ck.SourceKindFile && entry.Path != "" {
|
||||||
|
protected = append(protected, entry.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &processInputs{Configuration: loaded, Runtime: runtime, Source: source, RawTranscript: raw, ProtectedPaths: protected}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// preparedProcess groups the output resources and the facts needed for reports.
|
||||||
|
type preparedProcess struct {
|
||||||
|
Prepared *preflight.Prepared
|
||||||
|
RunID string
|
||||||
|
Started time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareProcess(ctx context.Context, input processInput, loaded *processInputs, stdout io.Writer) (*preparedProcess, error) {
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
reportDir := ""
|
||||||
|
reportDirSet := false
|
||||||
|
for _, override := range input.Overrides {
|
||||||
|
if override.Path.String() == "/reporting/stdout_dir" {
|
||||||
|
reportDirSet = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cfg.Reporting.Enabled {
|
||||||
|
reportDir = cfg.Reporting.StdoutDir
|
||||||
|
if input.Output.Value != "" && !reportDirSet {
|
||||||
|
reportDir, _ = filepath.Split(input.Output.Value)
|
||||||
|
if reportDir == "" {
|
||||||
|
reportDir = "."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runID, err := generateRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
started := time.Now().UTC()
|
||||||
|
var initial diagnostics.ReportBundle
|
||||||
|
var failed diagnostics.ReportRenderer
|
||||||
|
if cfg.Reporting.Enabled {
|
||||||
|
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
|
||||||
|
for index, item := range cfg.Pipeline.Stages {
|
||||||
|
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
|
||||||
|
}
|
||||||
|
initial, err = renderReportBundle(snapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
failed = func(primary error, failures []diagnostics.FileFailure) (diagnostics.ReportBundle, error) {
|
||||||
|
failedSnapshot := snapshot
|
||||||
|
completed := time.Now().UTC()
|
||||||
|
failedSnapshot.CompletedAt = &completed
|
||||||
|
return renderReportBundle(processreport.Failed(failedSnapshot, primary, "initialization", failures))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var debug *preflight.DebugInput
|
||||||
|
var warnings []error
|
||||||
|
if input.Debug.Value {
|
||||||
|
var effective bytes.Buffer
|
||||||
|
if err := loaded.Configuration.WriteEffective(&effective); err != nil {
|
||||||
|
warnings = append(warnings, err)
|
||||||
|
}
|
||||||
|
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||||
|
}
|
||||||
|
checkedBackends := make(map[string]struct{})
|
||||||
|
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
|
||||||
|
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
|
||||||
|
if errors.Is(err, pk.ErrProfileNotFound) {
|
||||||
|
return &preflight.SelectionError{Cause: err}
|
||||||
|
}
|
||||||
|
if err != nil || profile.Backend == nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
backend := *profile.Backend
|
||||||
|
if _, ok := checkedBackends[backend]; ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err = loaded.Runtime.Engine.InspectBackend(ctx, backend)
|
||||||
|
if errors.Is(err, pk.ErrBackendNotFound) {
|
||||||
|
return &preflight.SelectionError{Cause: err}
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
checkedBackends[backend] = struct{}{}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}, Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prepared.Warnings = append(warnings, prepared.Warnings...)
|
||||||
|
return &preparedProcess{Prepared: prepared, RunID: runID, Started: started}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateRunID() (string, error) {
|
||||||
|
var random [12]byte
|
||||||
|
if _, err := rand.Read(random[:]); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "run-" + hex.EncodeToString(random[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderReportBundle(value reporting.Snapshot) (diagnostics.ReportBundle, error) {
|
||||||
|
errorsYAML, err := reporting.YAML(value.ErrorsDocument())
|
||||||
|
if err != nil {
|
||||||
|
return diagnostics.ReportBundle{}, err
|
||||||
|
}
|
||||||
|
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument())
|
||||||
|
if err != nil {
|
||||||
|
return diagnostics.ReportBundle{}, err
|
||||||
|
}
|
||||||
|
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value)}, nil
|
||||||
|
}
|
||||||
108
internal/cli/process_inputs_test.go
Normal file
108
internal/cli/process_inputs_test.go
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestQuietDestinationCheckPrecedesInputAcquisition(t *testing.T) {
|
||||||
|
_, err := loadProcessInputs(context.Background(), processInput{Transcript: "missing.json"}, commandkit.ReportingModeQuiet)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "/dev/null") {
|
||||||
|
t.Fatalf("error=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessAcquisitionCapturesConsumedFiles(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"prompts.yml": "prompts:\n - id: extra\n messages:\n - role: user\n content_file: text.txt\n output:\n format: text\n validation_mode: none\n",
|
||||||
|
"text.txt": "example",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
|
||||||
|
result, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"config.yml", "prompts.yml", "text.txt"} {
|
||||||
|
if !slices.Contains(result.ProtectedPaths, filepath.Join(dir, name)) {
|
||||||
|
t.Errorf("missing %s in %v", name, result.ProtectedPaths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareProcessInitializesReportsAndToleratesDebugLoss(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
for name, contents := range map[string]string{
|
||||||
|
"config.yml": "version: 3\npipeline:\n stages: []\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
"blocked": "not a directory",
|
||||||
|
} {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json"), Output: commandkit.Supplied[string]{Value: filepath.Join(dir, "output.json"), Set: true}, Debug: commandkit.Supplied[bool]{Value: true, Set: true}, Overrides: []ck.Override{{Path: ck.Keys("diagnostics", "work_dir"), Value: filepath.Join(dir, "blocked"), Flag: "--work-dir"}}}
|
||||||
|
loaded, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := prepareProcess(context.Background(), input, loaded, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
if len(prepared.Prepared.Warnings) == 0 {
|
||||||
|
t.Fatal("debug loss was not reported")
|
||||||
|
}
|
||||||
|
report, err := os.ReadFile(filepath.Join(dir, "errors.yml"))
|
||||||
|
if err != nil || !strings.Contains(string(report), "in_progress") {
|
||||||
|
t.Fatalf("report=%q err=%v", report, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(input.Output.Value); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("output published early: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectionFlagJSON(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
raw string
|
||||||
|
valid bool
|
||||||
|
null bool
|
||||||
|
size int
|
||||||
|
}{
|
||||||
|
{`["party","spells"]`, true, false, 2}, {`[]`, true, false, 0}, {`null`, true, true, 0},
|
||||||
|
{`"party"`, false, false, 0}, {`[1]`, false, false, 0}, {`["x"] true`, false, false, 0},
|
||||||
|
} {
|
||||||
|
result, err := parseCollectionNames(tc.raw)
|
||||||
|
if (err == nil) != tc.valid {
|
||||||
|
t.Fatalf("%s error=%v", tc.raw, err)
|
||||||
|
}
|
||||||
|
if !tc.valid {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if tc.null {
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("null=%v", result)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
list, ok := result.([]any)
|
||||||
|
if !ok || len(list) != tc.size {
|
||||||
|
t.Fatalf("%s result=%v", tc.raw, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
245
internal/cli/process_ordered_integration_test.go
Normal file
245
internal/cli/process_ordered_integration_test.go
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/llm"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type controlledTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var models []string
|
||||||
|
var prompts []string
|
||||||
|
proposalCalls := 0
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var contents string
|
||||||
|
for _, message := range request.Messages {
|
||||||
|
contents += message.Content
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
models = append(models, request.Model)
|
||||||
|
prompts = append(prompts, contents)
|
||||||
|
if request.Model == "model-one" {
|
||||||
|
proposalCalls++
|
||||||
|
}
|
||||||
|
firstProposalCall := proposalCalls
|
||||||
|
mu.Unlock()
|
||||||
|
result := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||||
|
switch request.Model {
|
||||||
|
case "model-one":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice"}]}`
|
||||||
|
if firstProposalCall == 1 {
|
||||||
|
result = `{"invalid":true}`
|
||||||
|
}
|
||||||
|
case "model-two":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield"}]}`
|
||||||
|
case "model-three":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast"}]}`
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(result)
|
||||||
|
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||||
|
})
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": `version: 3
|
||||||
|
glossary_collections:
|
||||||
|
party:
|
||||||
|
entries: [{id: alice, name: Alice, category: person}]
|
||||||
|
spells:
|
||||||
|
entries: [{id: shield, name: Shield, category: spell}]
|
||||||
|
protected_terms: [party, spells]
|
||||||
|
llm: {validation_profile: review}
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
id: "party/../|<tag>"
|
||||||
|
glossaries: [party]
|
||||||
|
llm: {proposal_profile: one}
|
||||||
|
- module: glossary
|
||||||
|
id: spell
|
||||||
|
glossaries: [spells]
|
||||||
|
llm: {proposal_profile: two}
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party, spells]
|
||||||
|
llm: {proposal_profile: three}
|
||||||
|
`,
|
||||||
|
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
loaded.Config.Diagnostics.WorkDir = dir
|
||||||
|
loaded.Config.Reporting.StdoutDir = dir
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
|
||||||
|
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
|
||||||
|
}
|
||||||
|
if len(prompts) > 4 && !strings.Contains(prompts[4], "Alice casts Sheeld.") {
|
||||||
|
t.Fatalf("second proposal missed earlier edit: %s", prompts[4])
|
||||||
|
}
|
||||||
|
if len(result.Snapshot.WorkUnits) != 3 || result.Snapshot.WorkUnits[0].StageID == nil || *result.Snapshot.WorkUnits[0].StageID != "party/../|<tag>" || result.Snapshot.WorkUnits[1].StageIndex != 1 || result.Snapshot.WorkUnits[2].StageID != nil {
|
||||||
|
t.Fatalf("work unit identities: %+v", result.Snapshot.WorkUnits)
|
||||||
|
}
|
||||||
|
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
|
||||||
|
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
|
||||||
|
}
|
||||||
|
type coordinates struct {
|
||||||
|
StageIndex int `json:"stage_index"`
|
||||||
|
StageID *string `json:"stage_id"`
|
||||||
|
}
|
||||||
|
expected := map[int]string{0: "party/../|<tag>", 1: "spell", 2: ""}
|
||||||
|
check := func(name string, entries []coordinates) {
|
||||||
|
t.Helper()
|
||||||
|
seen := make(map[int]bool)
|
||||||
|
for _, entry := range entries {
|
||||||
|
id, ok := expected[entry.StageIndex]
|
||||||
|
if !ok || (id == "" && entry.StageID != nil) || (id != "" && (entry.StageID == nil || *entry.StageID != id)) {
|
||||||
|
t.Fatalf("%s has wrong stage association: %+v", name, entry)
|
||||||
|
}
|
||||||
|
seen[entry.StageIndex] = true
|
||||||
|
}
|
||||||
|
if len(seen) != len(expected) {
|
||||||
|
t.Fatalf("%s missing stage associations: %v", name, seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactCorrectionLedger))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var ledger []coordinates
|
||||||
|
if err := json.Unmarshal(raw, &ledger); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check("ledger", ledger)
|
||||||
|
raw, err = os.ReadFile(filepath.Join(prepared.DebugPath(), diagnostics.ArtifactLLMExecution))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var calls struct {
|
||||||
|
Calls []struct {
|
||||||
|
Context struct {
|
||||||
|
coordinates
|
||||||
|
Role string
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &calls); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var locations []coordinates
|
||||||
|
roles := make(map[int]map[string]bool)
|
||||||
|
for _, call := range calls.Calls {
|
||||||
|
location := call.Context.coordinates
|
||||||
|
locations = append(locations, location)
|
||||||
|
if roles[location.StageIndex] == nil {
|
||||||
|
roles[location.StageIndex] = make(map[string]bool)
|
||||||
|
}
|
||||||
|
roles[location.StageIndex][call.Context.Role] = true
|
||||||
|
}
|
||||||
|
check("LLM observations", locations)
|
||||||
|
for index := range expected {
|
||||||
|
if !roles[index]["proposal"] || !roles[index]["validation"] {
|
||||||
|
t.Fatalf("stage %d missing role evidence: %v", index, roles[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||||
|
t.Fatalf("models=%v", models)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
|
||||||
|
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
|
||||||
|
})
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
loaded.Config.Reporting.StdoutDir = dir
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, source.Canonical(), func(error) {}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" || result.Snapshot.Errors[0].StageID == nil || *result.Snapshot.Errors[0].StageID != "failing" {
|
||||||
|
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
57
internal/cli/process_output.go
Normal file
57
internal/cli/process_output.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v4/internal/framework/processreport"
|
||||||
|
commandkit "go.maximumdirect.net/commandkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// finishProcess delivers the transcript and reports before emitting success information.
|
||||||
|
func finishProcess(inv commandkit.Invocation, input processInput, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
|
||||||
|
reporter := inv.Reporter()
|
||||||
|
for _, warning := range prepared.Prepared.Warnings {
|
||||||
|
_ = reporter.Warn("Debug artifact: " + warning.Error())
|
||||||
|
}
|
||||||
|
completed := time.Now().UTC()
|
||||||
|
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
|
||||||
|
snapshot.TranscriptPath, snapshot.OutputPath = input.Transcript, input.Output.Value
|
||||||
|
snapshot.Status = reporting.StatusCompleted
|
||||||
|
reportPaths := prepared.Prepared.Output.ReportPaths()
|
||||||
|
var render diagnostics.ReportRenderer
|
||||||
|
if len(reportPaths) > 0 {
|
||||||
|
render = func(cause error, failures []diagnostics.FileFailure) (diagnostics.ReportBundle, error) {
|
||||||
|
report := snapshot
|
||||||
|
if cause != nil || len(failures) > 0 {
|
||||||
|
phase := "processing"
|
||||||
|
if primary == nil {
|
||||||
|
phase = "publication"
|
||||||
|
}
|
||||||
|
report = processreport.Failed(snapshot, cause, phase, failures)
|
||||||
|
}
|
||||||
|
return renderReportBundle(report)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result := prepared.Prepared.Output.Finish(inv.Context, primary, transcript, render)
|
||||||
|
if result != nil {
|
||||||
|
var infeasible *chunking.InfeasibleTargetError
|
||||||
|
var outputFailure diagnostics.FileFailure
|
||||||
|
if inv.Context.Err() == nil && errors.As(result, &infeasible) && !errors.As(result, &outputFailure) {
|
||||||
|
return commandkit.Usage(infeasible.Error()).WithCause(result)
|
||||||
|
}
|
||||||
|
return commandkit.Failure(result.Error()).WithCause(result)
|
||||||
|
}
|
||||||
|
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
|
||||||
|
for _, path := range reportPaths {
|
||||||
|
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", path))
|
||||||
|
}
|
||||||
|
if prepared.Prepared.Debug != nil {
|
||||||
|
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", prepared.Prepared.Debug.Path()))
|
||||||
|
}
|
||||||
|
return commandkit.Success()
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user