125 Commits
v3.0.1 ... main

Author SHA1 Message Date
2be7bbc015 Adopt the updated repository policies 2026-11-11 14:44:06 +00:00
64e5d0d275 Record intentional editorial acceptance across modules 2026-11-11 14:19:06 +00:00
ca36c3e319 Outline prompt quality discussion roadmap 2026-11-11 14:05:20 +00:00
6d34602f0a Remove completed correction repair planning documents 2026-11-11 13:52:42 +00:00
891c204e8f Identify the protected alias affected by a proposed edit 2026-11-11 12:34:20 +00:00
73ef2c6083 Select protected spelling findings without sorting 2026-11-11 12:31:44 +00:00
04e0c2f913 Keep detailed correction repair history in debug runs 2026-11-11 12:31:08 +00:00
6b724fd3cc Index applied changes for repair outcomes 2026-11-11 12:29:42 +00:00
8c93f728c0 Index initial repair findings and compact filtering once 2026-11-11 12:28:42 +00:00
5d90478f70 Share section lookups and replacement previews across validators 2026-11-11 12:28:00 +00:00
45b42a2689 Preserve cache settings in repair conversation replay 2026-11-11 12:26:27 +00:00
8267eba6a5 Retain application skip reasons for repaired proposals 2026-11-11 12:25:15 +00:00
bcce4729e1 Attribute filtered revisions to their repair attempts 2026-11-11 12:24:32 +00:00
d3a9edb29d Record correction repair scaling findings 2026-11-11 12:18:54 +00:00
d3fbddce11 Record correction repair integration findings 2026-11-11 12:11:36 +00:00
e029730c79 Record correction repair review findings 2026-11-11 12:02:36 +00:00
aa8073cab1 Validate correction repair through the processing command 2026-11-11 04:47:43 +00:00
f4c638313e Retain repair diagnostics and correction lineage 2026-11-11 04:37:52 +00:00
735dfc1834 Report correction repair operations and outcomes 2026-11-11 04:31:44 +00:00
d927c61199 Repair rejected corrections in independent branches 2026-11-11 04:29:18 +00:00
a5003a1e52 Collect independent correction validation findings 2026-11-11 04:23:34 +00:00
9fd36001b4 Configure correction repair profiles and budgets 2026-11-11 04:20:23 +00:00
00848203d7 Carry proposal exchanges into repair replay 2026-11-11 04:18:05 +00:00
eb45d0b623 Add module correction repair prompts 2026-11-11 04:13:55 +00:00
bb81354e21 Define correction repair response and conversation replay 2026-11-11 04:11:34 +00:00
93a080059d Plan correction repair implementation 2026-11-11 04:00:47 +00:00
b036336e2a Clarify correction repair limits and reporting 2026-11-11 03:48:30 +00:00
4f525e07cc Record accepted correction repair decisions 2026-11-11 03:36:26 +00:00
d91bb658c9 Plan correction repair after validation 2026-11-11 00:05:04 +00:00
8946a82407 Remove completed roadmaps and implementation plans 2026-11-10 20:52:52 +00:00
791deca3cd Skip application for sections without approved edits 2026-11-10 20:50:50 +00:00
c86f4fa416 Record completion of the interrupted ledger fix 2026-11-10 20:49:53 +00:00
2140314d40 Index interrupted ledger evidence by stage and proposal 2026-11-10 20:49:38 +00:00
71a9b1f560 Skip report and debug work for disabled outputs 2026-11-10 20:48:32 +00:00
a9b7c37a7e Reuse validation costs when fitting and batching 2026-11-10 20:45:38 +00:00
c16d61e091 Reuse prepared glossary data across stage sections 2026-11-10 20:42:28 +00:00
8ad27e8368 Avoid repeated glossary selection and membership work 2026-11-10 20:39:00 +00:00
1fd95abe95 Reuse credential discovery and diagnostic redactors 2026-11-10 20:37:52 +00:00
bb63c2cb10 Collect prompt file paths from effective configuration 2026-11-10 20:35:35 +00:00
7710101da0 Keep match boundaries for exempt glossary names 2026-11-10 20:33:24 +00:00
50940083c8 Preserve transcript destinations that change type 2026-11-10 20:31:03 +00:00
8a2a19ccd3 Mask padded credentials as sent to providers 2026-11-10 20:29:59 +00:00
bbac312b32 Honor cancellation before transcript delivery 2026-11-10 20:28:41 +00:00
56d39fca80 Clarify migration and subprocess failure guidance 2026-11-10 20:27:26 +00:00
9a52c8c246 Remove retired reporting and diagnostic code 2026-11-10 20:26:43 +00:00
a0e45c8f92 Show stage labels as literal Markdown text 2026-11-10 20:24:18 +00:00
bae96e5dd5 Load prompt configuration comparison sides independently 2026-11-10 20:22:23 +00:00
803effc6ea Show configuration findings when processing cannot start 2026-11-10 20:21:11 +00:00
45e5b381f5 Separate glossary context from prompt protection 2026-11-10 20:18:47 +00:00
a8eb5aea4d Resolve linked prompts in their host context 2026-11-10 20:17:54 +00:00
af0f9ac960 Keep failed processing reports consistent 2026-11-10 20:16:59 +00:00
3eacb33eb0 Record the algorithmic audit findings 2026-11-10 20:12:35 +00:00
88e166e3b1 Record the library integration audit findings 2026-11-10 18:51:12 +00:00
fa4f891689 Record the library adoption review findings 2026-11-10 18:41:23 +00:00
2844ce7e6a Complete assembled validation and migrate remaining tests 2026-11-10 18:13:12 +00:00
ba1d5d2367 Document the new configuration and processing workflow 2026-11-10 17:52:14 +00:00
926fa26a31 Identify pipeline entries in reports and diagnostics 2026-11-10 17:37:41 +00:00
7efa07ee9d Run ordered processing with independent settings 2026-11-10 17:33:30 +00:00
89de2a7014 Derive independent glossary context and protection 2026-11-10 17:25:08 +00:00
d42e2095d7 Resolve pipeline settings and collection selections 2026-11-10 17:18:30 +00:00
0a21ee2b50 Compose glossary collections through configuration 2026-11-10 17:14:31 +00:00
199c1d1dbe Connect processing preflight and output reporting 2026-11-10 17:12:28 +00:00
852e79405f Prepare and finalize output resources safely 2026-11-10 17:03:10 +00:00
8e1e073731 Enable prompt preview validation and smoke commands 2026-11-10 16:59:28 +00:00
8cc298d092 Adopt native command routing and inspection 2026-11-10 16:55:56 +00:00
50257ac554 Use native request execution and credential masking 2026-11-10 16:50:29 +00:00
497a1a18c5 Load bundled and operator prompt resources 2026-11-10 16:13:53 +00:00
7c64592cab Introduce the version three configuration loader 2026-11-10 16:08:53 +00:00
b500b855c6 Update toolchain and release integration 2026-11-10 16:02:17 +00:00
40420a217a Align adoption plans with main-relative loading releases 2026-11-10 15:49:05 +00:00
0c7622375f Plan the Audita adoption sprint 2026-11-09 16:03:16 +00:00
4920393204 Specify automatic PromptKit source discovery 2026-11-09 15:49:42 +00:00
866e4b9b23 Resolve output failures and preflight resource preparation 2026-11-09 15:25:57 +00:00
547be2d9ab Resolve ReleaseKit workflow and validator selection 2026-11-09 15:02:35 +00:00
f1fced498a Select CommandKit v0.6.0 and close the release gate 2026-11-09 14:18:34 +00:00
97f6f48098 Record remaining adoption and output-failure decisions 2026-11-09 14:02:56 +00:00
267aca4b76 Specify the Audita preflight package boundary 2026-11-09 13:37:25 +00:00
d539c06f45 Plan Audita preflight checks for empty glossary stages 2026-11-09 13:35:11 +00:00
5c69377563 Exempt active glossary spellings from overlapping protection 2026-11-09 13:22:16 +00:00
098401f674 Resolve optional stage IDs and reporting attribution 2026-11-09 12:55:46 +00:00
f016471880 Resolve glossary list clearing and configuration handoff 2026-11-09 12:53:18 +00:00
7e9bf6f149 Resolve glossary runtime views while retaining business logic 2026-11-09 12:06:53 +00:00
77995b7a2c Resolve extensible glossary entry content fields 2026-11-09 03:28:30 +00:00
1b76a426cc Resolve collection-scoped glossary entry identities 2026-11-09 02:45:50 +00:00
607fefac47 Select optional glossary file and directory underlay imports 2026-11-09 02:41:11 +00:00
6908dcd206 Document shared glossary libraries and campaign composition 2026-11-08 22:59:55 +00:00
42930d4921 Select named glossary collections with recursive directory imports 2026-11-08 21:07:15 +00:00
64786140e8 Plan ConfigKit glossary composition and derived runtime views 2026-11-08 18:51:37 +00:00
af6f62e14b Resolve glossary paths at the ConfigKit application handoff 2026-11-08 17:09:52 +00:00
167d91131e Exempt active glossary vocabulary from stage protection 2026-11-08 16:26:08 +00:00
4019f57eaa Plan separate glossary and protection lists with root defaulting 2026-11-08 16:11:32 +00:00
760a7cecc0 Resolve stage CLI precedence within native ConfigKit capabilities 2026-11-08 15:34:20 +00:00
943152d1d5 Resolve compound stage settings and explicit clearing rules 2026-11-08 15:27:39 +00:00
c94086048a Adopt upstream ADR guidance and revise durable documentation requirements 2026-11-08 15:18:38 +00:00
d2c0e22a9a Resolve stage setting scope and validation profile fallback 2026-11-08 15:07:13 +00:00
0b3734d352 Plan root defaults and independent stage overrides 2026-11-08 14:58:37 +00:00
b7bad87659 Select application schema version 3 with a hard cutover 2026-11-08 14:48:51 +00:00
c46a689e29 Resolve standard request commands and upstream documentation ownership 2026-11-08 14:42:58 +00:00
9dba2d506a Require an explicit transcript destination for quiet processing 2026-11-08 14:31:44 +00:00
68e16f6aac Resolve quiet reporting and completion summary behavior 2026-11-08 14:05:00 +00:00
1f2beef08c Gate CommandKit adoption on released shared debug binding 2026-11-08 13:56:40 +00:00
32bac58b5e Separate debug artifact capture from reporting mode 2026-11-08 13:54:35 +00:00
90d82b5b5f Resolve unified debug mode and record CommandKit API gap 2026-11-08 13:25:44 +00:00
a9dba890ba Resolve behavior when configuration discovery finds no file 2026-11-08 05:20:05 +00:00
04f9f16e27 Resolve configuration discovery order and import support 2026-11-08 05:17:43 +00:00
893f66b34b Resolve retirement of the curated profile summary 2026-11-08 05:14:48 +00:00
98ffb67432 Plan namespaced bundled prompts and operator replacements 2026-11-08 02:13:03 +00:00
ec8dffc06e Resolve native backend and credential ownership 2026-11-08 02:05:48 +00:00
d4d510a520 Resolve backend concurrency ownership and plan redundant code removal 2026-11-08 01:48:50 +00:00
ac86c77cfe Plan stage-specific glossary inputs for the next major release 2026-11-08 01:42:48 +00:00
259f728f20 Plan ReleaseKit adoption and a major-release hard cutover 2026-11-08 01:40:10 +00:00
46765bd326 Document native library adoption plans 2026-11-08 01:31:18 +00:00
cbcb1b3778 Index the v3.1.1 release notes
Some checks failed
ci/woodpecker/tag/release Pipeline failed
2026-10-11 01:56:54 +00:00
39052bf47e Write the Audita v3.1.1 release note 2026-10-11 01:55:06 +00:00
7cd5edd650 Resolve annotated tag events before CI delivery 2026-10-11 01:51:44 +00:00
2d41748006 Use the Go v3 module path for Audita
Some checks failed
ci/woodpecker/tag/release Pipeline failed
2026-10-11 01:40:38 +00:00
1d4be9988e Index the v3.1.0 release notes 2026-10-11 01:38:14 +00:00
ed65aa6474 Remove completed roadmaps and preserve deferred work 2026-10-11 01:37:33 +00:00
4c5536ab78 Write the Audita v3.1.0 release notes 2026-10-11 01:36:37 +00:00
b01180ea83 Upgrade ConfigKit, ReleaseKit, and PromptKit 2026-10-11 01:35:09 +00:00
07fab5586b Document the latest library updates and adoption work 2026-10-11 01:25:53 +00:00
6debe7e641 Upgrade ReleaseKit and simplify release tooling 2026-10-09 04:09:26 +00:00
11c349d9ae Adopt the updated release policy 2026-10-09 03:53:18 +00:00
36823c91fb Plan the ReleaseKit v0.3.0 upgrade 2026-10-09 03:44:10 +00:00
17421bc5f3 Use the tagged module selection for CI delivery 2026-10-07 23:45:27 +00:00
253 changed files with 10176 additions and 7454 deletions

View File

@@ -1,7 +1,6 @@
matrix:
include:
- GO_IMAGE: golang:1.25.5
RELEASEKIT_VERSION: v0.1.0
- GO_IMAGE: golang:1.27.3-bookworm
when:
- event: tag
@@ -15,7 +14,38 @@ steps:
from_secret: GITEA_RELEASE_TOKEN
commands:
- |
go run gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit@${RELEASEKIT_VERSION} ci-deliver \
set -eu
test -n "$${CI_COMMIT_TAG}"
test -n "$${CI_COMMIT_SHA}"
git check-ref-format "refs/tags/$${CI_COMMIT_TAG}"
git fetch origin "refs/tags/$${CI_COMMIT_TAG}:refs/tags/$${CI_COMMIT_TAG}"
tag_commit=$(git rev-parse "refs/tags/$${CI_COMMIT_TAG}^{commit}")
expected_commit=$(git rev-parse --verify "$${CI_COMMIT_SHA}^{commit}")
test "$(printf '%s' "$${tag_commit}" | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
git checkout --detach "$${tag_commit}"
test "$(git rev-parse HEAD | tr '[:upper:]' '[:lower:]')" = "$${expected_commit}"
test -z "$(git status --porcelain --untracked-files=all)"
bootstrap_dir=$(mktemp -d "$${CI_WORKSPACE%/*}/.releasekit-bootstrap.XXXXXX")
trap 'rm -rf "$${bootstrap_dir}"' EXIT
cp go.mod "$${bootstrap_dir}/releasekit.mod"
if [ -f go.sum ]; then
cp go.sum "$${bootstrap_dir}/releasekit.sum"
else
: > "$${bootstrap_dir}/releasekit.sum"
fi
export GOWORK=off
export GOTOOLCHAIN=local
releasekit_host_os=$(go env GOHOSTOS) || exit $?
releasekit_host_arch=$(go env GOHOSTARCH) || exit $?
export GOOS="$${releasekit_host_os}" GOARCH="$${releasekit_host_arch}"
releasekit_module=go.maximumdirect.net/releasekit
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -m "$${releasekit_module}"
go mod edit -modfile="$${bootstrap_dir}/releasekit.mod" -tool="$${releasekit_module}/cmd/releasekit"
go list -mod=mod -modfile="$${bootstrap_dir}/releasekit.mod" -deps "$${releasekit_module}/cmd/releasekit" > /dev/null
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
--repo "$${CI_WORKSPACE}" \
--commit "$${CI_COMMIT_SHA}" \
--prepared \
--commit "$${expected_commit}" \
"$${CI_COMMIT_TAG}"

View File

@@ -1,55 +1,34 @@
# Audita
Audita polishes transcript JSON using glossary-aware and LLM-backed correction
modules. It validates proposed corrections and produces corrected text plus
reports for review and subprocess integration.
Audita polishes transcript JSON with glossary-aware and LLM-backed correction
modules. It validates proposed edits and produces corrected text and reports for
review or subprocess integration.
## Quickstart
From a checkout, build with Go 1.25.5 or newer:
Build with Go 1.27.0 or newer:
```sh
go build -o ./bin/audita ./cmd/audita
```
Processing requires a reachable [compatible LLM endpoint](docs/integrations/openai-compatible-llm.md)
and access to a model you select. The [minimal config](examples/minimal-config.yml)
targets an unauthenticated local endpoint; edit its endpoint and
[local profiles](examples/profiles/local/) to match your service and models. The
[service example](examples/production-config.yml) demonstrates a named credential
reference and a shared backend. Set its referenced environment variable before
validating it. The [testing overlay](examples/overlays/testing.yml) selects both
role profiles, but model cost, speed, and quality depend on your provider choices.
Check configuration and profile selection offline:
Edit the synthetic endpoint and model in [examples/local/prompts.yml](examples/local/prompts.yml)
for a reachable OpenAI-compatible service. From `examples/local`, run the built
executable (adjust the executable path if needed):
```sh
./bin/audita config validate --config examples/minimal-config.yml
./bin/audita config print-effective --config examples/minimal-config.yml --view profiles
../../bin/audita config validate --config config.yml
../../bin/audita process --config config.yml --output corrected.json transcript.json
```
Run the included small input with the minimal config:
```sh
./bin/audita process examples/tiny-transcript.json \
--config examples/minimal-config.yml \
--glossary examples/tiny-glossary.yaml \
--output corrected.json
```
This makes LLM requests. Explicit setting flags override the selected file and
optional overlay; general environment setting overrides are no longer read.
By default the command writes `errors.yml`, `recoveries.yml`, and `report.md`
beside `corrected.json`. Use `--reporting=false` to suppress those files, or
`--debug` to retain detailed run artifacts under the configured debug work
directory. Model failures affecting some operations can still yield a completed
transcript with applied corrections; the summary and reports identify them.
See the [v2 migration guide](docs/migrations/v2.md) for old configurations.
Processing makes model requests. Reports are written beside the output by default;
recoverable model errors can appear in a completed run. The [example guide](examples/README.md)
includes service, overlay, prompt editing, preview, and shared glossary projects.
See the [v3 migration guide](docs/migrations/v3.md) for the configuration cutover.
## Documentation
- [CLI reference](docs/cli.md)
- [Configuration and maintained examples](docs/config.md)
- [CLI reference](docs/cli.md) and [configuration](docs/config.md)
- [Operations and artifact lifecycle](docs/operations.md)
- [Transcript and glossary inputs](docs/integrations/transcript-glossary-files.md)
- [Output files and report formats](docs/integrations/output-files.md)

View File

@@ -0,0 +1,374 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
)
type repairWireMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type repairWireRequest struct {
Messages []repairWireMessage `json:"messages"`
ResponseFormat struct {
JSONSchema struct {
Schema struct {
Properties map[string]any `json:"properties"`
} `json:"schema"`
} `json:"json_schema"`
} `json:"response_format"`
}
type correctionProvider struct {
mu sync.Mutex
requests []repairWireRequest
attempts map[int]int
entered chan struct{}
wait bool
invalidFirstRevision bool
}
func (p *correctionProvider) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var req repairWireRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "bad request", 400)
return
}
_, repair := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]
_, validation := req.ResponseFormat.JSONSchema.Schema.Properties["validations"]
p.mu.Lock()
p.requests = append(p.requests, req)
p.mu.Unlock()
content := `{"corrections":[{"id":1,"original_text":"hello","corrected_text":"Hello","confidence":0.99},{"id":2,"original_text":"second","corrected_text":"Second","confidence":0.1},{"id":3,"original_text":"third","corrected_text":"Third","confidence":0.1}]}`
if validation {
var indices []int
for _, message := range req.Messages {
if pos := strings.LastIndex(message.Content, "Corrections to validate:"); pos >= 0 {
var items []struct {
CorrectionIndex int `json:"correction_index"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[pos+len("Corrections to validate:"):])), &items); err != nil {
http.Error(w, "bad validation", 400)
return
}
for _, item := range items {
indices = append(indices, item.CorrectionIndex)
}
}
}
var decisions []map[string]any
for _, index := range indices {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": true, "confidence": 0.99, "reason": "controlled"})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
}
if repair {
if p.wait {
select {
case p.entered <- struct{}{}:
default:
}
<-r.Context().Done()
return
}
ref := 2
if strings.Contains(req.Messages[len(req.Messages)-1].Content, `"proposal_ref":3`) {
ref = 3
}
p.mu.Lock()
p.attempts[ref]++
attempt := p.attempts[ref]
invalid := p.invalidFirstRevision && attempt == 1
p.mu.Unlock()
if ref == 3 {
http.Error(w, `{"error":{"message":"context too long"}}`, http.StatusBadRequest)
return
}
confidence := 0.1
if attempt > 1 {
confidence = 0.99
}
id := 2
if invalid {
id, confidence = 99, 0.99
}
content = fmt.Sprintf(`{"proposal_ref":2,"action":"revise","correction":{"id":%d,"original_text":"second","corrected_text":"Second","confidence":%v}}`, id, confidence)
}
raw, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"model":"fixture","choices":[{"message":{"content":%s}}]}`, raw)
}
func TestSubprocessReportsDefectOnOriginatingRepair(t *testing.T) {
for _, tc := range []struct {
budget int
outcome string
recoveries int
}{{1, "exhausted", 1}, {2, "applied", 2}} {
t.Run(tc.outcome, func(t *testing.T) {
f, provider := setupCorrectionFixture(t, tc.budget, false, false)
provider.mu.Lock()
provider.invalidFirstRevision = true
provider.mu.Unlock()
if code, _, stderr := f.run(t, true); code != 0 {
t.Fatalf("code=%d stderr=%s", code, stderr)
}
doc := f.recoveries(t)
if *doc.Summary.ProposalResponses != 1 || *doc.Summary.ProposalResponsesWithFiltering != 0 || len(doc.Recoveries) != tc.recoveries {
t.Fatalf("unexpected recoveries: %+v", doc)
}
first := doc.Recoveries[0]
want := []reporting.FilteredProposal{{ProposalIndex: 1, Reason: "invalid_target_segment_id", Count: 1}}
if first.Operation != "correction_repair" || !reflect.DeepEqual(first.FilteredProposals, want) || !reflect.DeepEqual(first.Reasons, []string{"proposals_filtered"}) {
t.Fatalf("first repair recovery: %+v", first)
}
if tc.recoveries == 2 && !reflect.DeepEqual(doc.Recoveries[1].Reasons, []string{"correction_revised"}) {
t.Fatalf("successful repair recovery: %+v", doc.Recoveries[1])
}
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil || !strings.Contains(string(report), fmt.Sprintf("| 1 | 1 | 1 | %d | %s |", tc.budget, tc.outcome)) {
t.Fatalf("repair outcome: %s %v", report, err)
}
})
}
}
func setupCorrectionFixture(t *testing.T, attempts int, replacement, wait bool) (processFixture, *correctionProvider) {
t.Helper()
f := newProcessFixture(t, "normal", "[grammar]")
f.setSegments(t, "hello world", "second text", "third text")
provider := &correctionProvider{attempts: map[int]int{}, entered: make(chan struct{}, 1), wait: wait}
server := httptest.NewServer(provider)
t.Cleanup(server.Close)
resources := fmt.Sprintf("profiles:\n - id: proposal\n endpoint: %s/v1\n model: fixture\n - id: validation\n endpoint: %s/v1\n model: fixture\n", server.URL, server.URL)
if replacement {
resources += "collections:\n audita:\n prompts:\n - id: modules.grammar.repair\n messages:\n - role: user\n content_file: repair.txt\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_repair.v1.json\n"
if err := os.WriteFile(filepath.Join(f.dir, "repair.txt"), []byte("OVERRIDE {{ .RepairRequestJSON }}"), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
data, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
text := strings.Replace(string(data), "correction_repair_attempts: 0", fmt.Sprintf("correction_repair_attempts: %d", attempts), 1)
text += "chunking:\n target_sections: 1\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
if err := os.WriteFile(f.config, []byte(text), 0600); err != nil {
t.Fatal(err)
}
return f, provider
}
func TestSubprocessCorrectionRepairConversation(t *testing.T) {
for _, tc := range []struct {
name string
debug, replacement bool
}{{"default prompt without debug", false, false}, {"replacement with debug", true, true}} {
t.Run(tc.name, func(t *testing.T) {
f, provider := setupCorrectionFixture(t, 2, tc.replacement, false)
args := []string{"--verbose"}
if tc.debug {
args = append(args, "--debug")
}
code, stdout, stderr := f.run(t, true, args...)
if code != 0 || stdout != "" {
t.Fatalf("code=%d stdout=%s stderr=%s", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
if len(output.Segments) != 3 || output.Segments[0].Text != "Hello world" || output.Segments[1].Text != "Second text" || output.Segments[2].Text != "third text" {
t.Fatalf("transcript: %+v", output)
}
report, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil {
t.Fatal(err)
}
for _, fragment := range []string{"## Correction repair", "| 1 | 1 | 1 | 2 | applied |", "| 1 | 1 | 2 | 1 | unavailable |"} {
if !strings.Contains(string(report), fragment) {
t.Errorf("missing %q in %s", fragment, report)
}
}
doc := f.errors(t)
recoveries := f.recoveries(t)
if doc.Summary.LLMOperations == nil || *doc.Summary.LLMOperations != 10 || *doc.Summary.CorrectionsApplied != 2 || *doc.Summary.Recoveries != 1 || !reflect.DeepEqual(doc.Summary, recoveries.Summary) || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("counts: errors=%+v recoveries=%+v stderr=%s", doc.Summary, recoveries.Summary, stderr)
}
if len(doc.Errors) != 1 || len(doc.Errors[0].Failures) != 1 || doc.Errors[0].Failures[0].Operation != "correction_repair" || doc.Errors[0].Failures[0].Reason != "provider_failed" {
t.Fatalf("errors: %+v", doc)
}
provider.mu.Lock()
requests := append([]repairWireRequest(nil), provider.requests...)
provider.mu.Unlock()
var base []repairWireMessage
var branches [][]repairWireMessage
for _, req := range requests {
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["corrections"]; ok {
base = req.Messages
}
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
branches = append(branches, req.Messages)
}
}
if len(base) != 2 || len(branches) != 3 || len(branches[0]) != 4 || len(branches[1]) != 6 || len(branches[2]) != 4 {
t.Fatalf("request lengths base=%d branches=%v", len(base), branches)
}
for _, messages := range branches {
for i := range base {
if messages[i] != base[i] {
t.Fatalf("changed prefix: %+v", messages)
}
}
if messages[2].Role != "assistant" || !strings.Contains(messages[2].Content, `"corrections"`) {
t.Fatalf("missing initial response: %+v", messages)
}
}
if branches[1][3] != branches[0][3] || branches[1][4].Role != "assistant" || strings.Contains(branches[2][3].Content, `"proposal_ref":2`) {
t.Fatalf("branch history: %+v", branches)
}
if tc.replacement && !strings.Contains(branches[0][3].Content, "OVERRIDE") {
t.Fatal("replacement not used")
}
if tc.debug {
_, ledger := f.debugArtifacts(t)
if len(ledger) != 3 {
t.Fatalf("ledger: %+v", ledger)
}
}
})
}
}
func TestSubprocessRepairDisabledAndOutputSelections(t *testing.T) {
for _, tc := range []struct {
name string
reporting, debug bool
}{{"neither", false, false}, {"debug only", false, true}, {"report only", true, false}} {
t.Run(tc.name, func(t *testing.T) {
f, provider := setupCorrectionFixture(t, 0, false, false)
if !tc.reporting {
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
t.Fatal(err)
}
fmt.Fprintln(file, "reporting:\n enabled: false")
file.Close()
}
var args []string
if tc.debug {
args = append(args, "--debug")
}
code, _, stderr := f.run(t, true, args...)
if code != 0 {
t.Fatalf("code=%d stderr=%s", code, stderr)
}
provider.mu.Lock()
requests := append([]repairWireRequest(nil), provider.requests...)
provider.mu.Unlock()
for _, req := range requests {
if _, ok := req.ResponseFormat.JSONSchema.Schema.Properties["proposal_ref"]; ok {
t.Fatal("repair request with zero budget")
}
}
_, err := os.Stat(filepath.Join(f.dir, "report.md"))
if tc.reporting != (err == nil) {
t.Fatalf("report selection: %v", err)
}
if tc.debug {
_, ledger := f.debugArtifacts(t)
if len(ledger) == 0 {
t.Fatal("debug ledger missing")
}
} else if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
t.Fatalf("unexpected debug dir: %v", err)
}
})
}
}
func TestSubprocessCancellationDuringCorrectionRepair(t *testing.T) {
f, provider := setupCorrectionFixture(t, 2, false, true)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output, "--debug"}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1")
var stderr bytes.Buffer
command.Stderr = &stderr
if err := command.Start(); err != nil {
t.Fatal(err)
}
select {
case <-provider.entered:
case <-ctx.Done():
t.Fatalf("repair not reached: %v", ctx.Err())
}
if err := command.Process.Signal(os.Interrupt); err != nil {
t.Fatal(err)
}
err := command.Wait()
if ctx.Err() != nil || err == nil {
t.Fatalf("cancellation: %v ctx=%v stderr=%s", err, ctx.Err(), stderr.String())
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("transcript published: %v", err)
}
doc := f.errors(t)
if doc.Status != reporting.StatusFailed || len(doc.Errors) == 0 {
t.Fatalf("failed report: %+v", doc)
}
data, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil || !strings.Contains(string(data), "| 1 | 1 | 1 | 1 | interrupted |") {
t.Fatalf("partial report %s %v", data, err)
}
_, ledger := f.debugArtifacts(t)
if len(ledger) == 0 {
t.Fatal("partial ledger missing")
}
}
func TestSubprocessRepairWithoutOrdinaryReporting(t *testing.T) {
for _, debug := range []bool{false, true} {
t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) {
f, _ := setupCorrectionFixture(t, 2, false, false)
file, err := os.OpenFile(f.config, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
t.Fatal(err)
}
fmt.Fprintln(file, "reporting:\n enabled: false")
file.Close()
args := []string{"--verbose"}
if debug {
args = append(args, "--debug")
}
code, _, stderr := f.run(t, true, args...)
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%s", code, stderr)
}
if _, err := os.Stat(filepath.Join(f.dir, "report.md")); !os.IsNotExist(err) {
t.Fatalf("unexpected report: %v", err)
}
if debug {
_, ledger := f.debugArtifacts(t)
if len(ledger) != 3 {
t.Fatalf("ledger: %+v", ledger)
}
}
})
}
}

View File

@@ -6,7 +6,7 @@ import (
"os/signal"
"syscall"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/v3/internal/cli"
)
func main() {

View File

@@ -19,9 +19,9 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/cli"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gopkg.in/yaml.v3"
)
@@ -122,6 +122,11 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
return
}
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
if p.mode == "invalid-credential-echo" {
sent := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ")
raw, _ := json.Marshal(map[string]string{"unexpected": sent + "|" + p.secret})
content = string(raw)
}
if p.mode == "repair-filter-later-failure" && !validation {
if proposalCall == 1 {
content = `{"wrong":true}`
@@ -231,17 +236,15 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
t.Helper()
dir := t.TempDir()
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
if mode == "invalid-credential-echo" {
p.secret = " synthetic-echo-credential\t "
}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
for _, id := range []string{"proposal", "validation"} {
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
}
configPath := filepath.Join(dir, "config.yml")
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
@@ -250,7 +253,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
var stageList []string
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
if name = strings.TrimSpace(name); name != "" {
stageList = append(stageList, " - module: "+name)
}
}
pipeline := " stages: []\n"
if len(stageList) != 0 {
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
}
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n correction_repair_attempts: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
@@ -271,7 +284,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
t.Helper()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
if includeOutput {
args = append(args, "--output", f.output)
}
@@ -775,7 +788,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
if code != 0 || stdout != "" || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
@@ -807,7 +820,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
@@ -830,7 +843,7 @@ func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testin
}
f.output = filepath.Join(transcriptDir, "corrected.json")
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
@@ -858,43 +871,35 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
for _, selection := range []struct{ reporting, debug bool }{
{false, false}, {false, true}, {true, false}, {true, true},
} {
f := newProcessFixture(t, "normal", "[grammar]")
if selection.reporting && selection.debug {
original, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
updated := strings.Replace(string(original), old, inherited, 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
t.Fatal(err)
}
}
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
reportDir := filepath.Join(f.dir, "missing")
extra := []string{"--reporting=false", "--report-dir", reportDir}
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
if selection.reporting {
reportDir = filepath.Join(f.dir, "selected-reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
extra = []string{"--reporting=true", "--report-dir", reportDir}
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
}
if selection.debug {
extra = append(extra, "--debug")
}
code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
summary := strings.SplitN(stderr, "\n", 2)[0]
summary := ""
for _, line := range strings.Split(stderr, "\n") {
if strings.HasPrefix(line, "Completed:") {
summary = line
break
}
}
if expectedOutput == "" {
expectedOutput, expectedSummary = string(output), summary
} else if string(output) != expectedOutput || summary != expectedSummary {
@@ -916,31 +921,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
t.Fatal(err)
}
}
if selection.reporting {
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
t.Fatalf("effective inherited settings: %s", effective)
}
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
if !strings.Contains(string(origins), want) {
t.Fatalf("configuration origins missing %q: %s", want, origins)
}
}
if strings.Contains(string(origins), f.provider.secret) {
t.Fatal("configuration origins exposed the resolved inherited credential")
}
code, explain, stderr := f.runConfigView(t, "explain")
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
}
}
}
}
}
@@ -954,7 +935,7 @@ func boolInt(value bool) int {
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -971,9 +952,41 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
}
}
func TestSubprocessMasksPaddedCredentialInProviderResponse(t *testing.T) {
f := newProcessFixture(t, "invalid-credential-echo", "[grammar]")
effective := strings.TrimSpace(f.provider.secret)
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || f.provider.callCount() != 1 || !strings.Contains(stderr, "1 work units with errors") {
t.Fatalf("code=%d calls=%d stdout=%q stderr=%q", code, f.provider.callCount(), stdout, stderr)
}
if strings.Contains(stderr, effective) || strings.Contains(stdout, effective) {
t.Fatal("credential escaped redaction in command output")
}
responseCaptured := false
err := filepath.WalkDir(f.dir, func(path string, entry os.DirEntry, err error) error {
if err != nil || entry.IsDir() {
return err
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
if bytes.Contains(data, []byte(effective)) {
t.Errorf("credential escaped redaction in %s", entry.Name())
}
if strings.HasSuffix(entry.Name(), "-response-payload.json") && bytes.Contains(data, []byte("[REDACTED]|[REDACTED]")) {
responseCaptured = true
}
return nil
})
if err != nil || !responseCaptured {
t.Fatalf("missing redacted provider echo: %v", err)
}
}
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -989,7 +1002,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
}
@@ -1019,7 +1032,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1037,7 +1050,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stderr bytes.Buffer
@@ -1067,7 +1080,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1081,55 +1094,41 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
}
}
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
}
code, _, _ := f.run(t, true)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failed transcript published: %v", err)
}
var recoveries reporting.RecoveriesDocument
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
t.Fatal(err)
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
t.Fatal(err)
}
if recoveries.Status != reporting.StatusFailed {
t.Fatalf("recoveries status: %+v", recoveries)
}
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
}
}
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
original, err := os.ReadFile(f.config)
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
}
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
@@ -1139,25 +1138,13 @@ func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing
t.Fatal(err)
}
code, _, _ := f.run(t, true, "--report-dir", reportDir)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
if err != nil {
t.Fatal(err)
}
var document reporting.ErrorsDocument
if err := yaml.Unmarshal(raw, &document); err != nil {
t.Fatal(err)
}
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
t.Fatal(err)
}
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
t.Fatalf("report created before preflight: %s %v", name, err)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("failed report appeared in default directory: %v", err)
@@ -1182,7 +1169,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
}
if _, err := os.Stat(f.output); err != nil {
@@ -1197,7 +1184,7 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
f.output = filepath.Join(f.dir, "missing", "output.json")
code, _, stderr := f.run(t, true, "--reporting=false")
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
}
@@ -1256,12 +1243,12 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
selected, unused := "selected-path-secret", "unused-path-secret"
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(f.config)
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
@@ -1276,20 +1263,20 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
}
}
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
if (!fail && code != 0) || (fail && code != 1) {
if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
t.Fatalf("credential in stderr: %q", stderr)
}
if !strings.Contains(stderr, "[REDACTED]") {
if !fail && !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err)
}
dirs, err := os.ReadDir(debugParent)
if err != nil || len(dirs) != 1 {
if !fail && (err != nil || len(dirs) != 1) {
t.Fatalf("debug location: %v %v", dirs, err)
}
if !fail {
@@ -1303,7 +1290,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, baselineSummary := f.run(t, true)
code, _, baselineSummary := f.run(t, true, "--verbose")
if code != 0 {
t.Fatal(baselineSummary)
}
@@ -1318,16 +1305,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
t.Errorf("debug directory: %v %v", dirs, err)
return
}
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
t.Error(err)
}
}
f.provider.mu.Unlock()
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
t.Fatalf("changed outcome: %d %q", code, stderr)
}
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
t.Fatalf("missing debug failures: %q", stderr)
}
output, err := os.ReadFile(f.output)
@@ -1360,7 +1347,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) {
t.Fatal(err)
}
if invalid {
if code != 1 || !bytes.Equal(before, after) {
if code != 2 || !bytes.Equal(before, after) {
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
}
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
@@ -1374,15 +1361,18 @@ func TestSubprocessProfileSources(t *testing.T) {
f := newProcessFixture(t, "success", "[]")
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
t.Fatal(err)
}
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
code, _, stderr := f.run(t, true, "--reporting=false")
if code != 0 {
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
@@ -1396,12 +1386,12 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
}
parentDir := filepath.Join(f.dir, "profiles", "catalog")
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
parentDir := filepath.Join(f.dir, "prompts", "catalog")
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
@@ -1411,8 +1401,8 @@ func TestSubprocessProfileSources(t *testing.T) {
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "report.md")
for path, content := range map[string]string{
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
child: "prompts:\n literal/~child:\n extends: source\n",
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
child: "prompts:\n - id: child\n extends: {id: source}\n",
resource: "Captured operator message.\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
@@ -1429,7 +1419,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
}
@@ -1439,7 +1429,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
}
}

View File

@@ -0,0 +1,42 @@
# ADR 0002: Glossary configuration and processing readiness
**Status:** Accepted
**Date:** 2026-11-10
## Context
Operators need to reuse glossary collections across campaigns and select different
context and protected terms for each ordered correction. Configuration inspection
must remain useful without a transcript, model credentials, or a populated glossary.
A processing run must reject an unusable later correction before publishing outputs
or spending resources on earlier corrections.
## Decision
ConfigKit composes typed, named glossary collections with the application
configuration. Each pipeline entry selects context and protection independently;
Audita derives consumer views without changing the resolved configuration. For a
glossary correction, active recognized spellings are exempt from that entry's
protection view. Other protected spellings and validators remain effective.
After configuration resolution, Audita preflight checks the whole pipeline and
coordinates output preparation. It consumes the typed configuration and acquired
inputs, without reloading configuration or calling a provider. The output session
owns preparation, cleanup, and publication. See the [configuration](../config.md)
and [operations](../operations.md) contracts for current behavior.
## Alternatives considered
A separately parsed glossary per invocation would duplicate composition and make
per-entry selection and provenance harder to express. Requiring nonempty
collections in configuration validation would prevent inspection and reuse of
valid empty definitions. Preparing outputs or checking each entry only when it
runs could leave artifacts or incur model work before finding a later error.
## Consequences
Configuration and runtime views have distinct responsibilities. Inspection can
show configured selections and origins without presenting derived vocabulary as
configuration. Preflight and the output session need a clear handoff, and the
processing pipeline must carry each entry's own effective settings and identity.

View File

@@ -0,0 +1,59 @@
# ADR 0003: Module focus and editorial acceptance
**Status:** Accepted
**Date:** 2026-11-11
## Context
Audita gives each correction module a focused task. A prompt review suggested
making that task's scope an acceptance condition, because validators can accept
minor editorial changes that the proposing module was not asked to make.
The maintainer's practical testing established why this difference is deliberate.
Models persistently propose minor editorial cleanup when it is clearly warranted
in the supplied text. Instructions strict enough to suppress that cleanup also
suppress desirable edits. Enforcing exclusive module scopes would therefore
trade away useful corrections to achieve a separation that does not serve the
application's editorial goals.
## Decision
Module scope guides the corrections Audita asks a model to propose. It is not an
exhaustive boundary on acceptable edits. Minor editorial cleanup is permitted
across modules and their validators even when the proposal prompt does not
request it.
Validators must not reject an otherwise acceptable minor editorial change solely
because it falls outside the originating module's requested focus. Existing
checks for applicability, protected terms, meaning, and other domain concerns
continue to apply. This allowance does not guarantee that any particular edit
passes validation or application, nor does it authorize arbitrary rewriting.
Proposal, validation, and repair instructions may differ to serve their roles.
Their consistency should be assessed against this distinction, rather than by
requiring identical lists of allowed edits. Repair feedback should not turn
module focus into an exclusive acceptance rule.
## Alternatives considered
**Strict module-aware acceptance**, whether implemented in shared validators,
separate review definitions, or an additional scope check, would make stage
boundaries easier to describe. It is rejected because the maintainer's testing
shows that suppressing incidental cleanup also suppresses desirable work.
**Request every kind of cleanup in every module** would align proposal requests
with the broader acceptance policy, but would dilute the focused tasks. Permitting
incidental cleanup does not require asking each stage to perform every task.
## Consequences
An edit accepted outside a module's requested focus is not, by itself, a design
defect. Prompt maintenance and evaluation should preserve useful corrections
while checking substantive editorial quality. They should not introduce strict
scope enforcement as an apparent consistency fix.
The [architecture policy](../policy/architecture.md#correction-and-concurrency-invariants)
records the boundary. Reconsidering this accepted decision requires new evidence
and explicit supersession under the ADR lifecycle, rather than rediscovering the
same prompt/validator difference during each review.

View File

@@ -4,6 +4,10 @@
- [ADR 0001: Adopt ReleaseKit for release management](0001-releasekit.md) —
accepted 2026-10-07.
- [ADR 0002: Glossary configuration and processing readiness](0002-glossary-configuration-and-preflight.md) —
accepted 2026-11-10.
- [ADR 0003: Module focus and editorial acceptance](0003-module-focus-and-editorial-acceptance.md) —
accepted 2026-11-11.
Use the
[documentation policy's ADR lifecycle](../policy/documentation.md#architecture-decision-records)

View File

@@ -1,116 +1,45 @@
# Audita CLI Reference
The shared version-2 configuration parser and offline profile inspection serve
`process`, `config validate`, and `config print-effective`.
The [configuration reference](config.md) owns fields, defaults, and validation;
the [migration guide](migrations/v2.md) maps retired inputs.
Audita uses CommandKit for routing and the ConfigKit and PromptKit integrations
for their standard inspection and request commands. Run `audita --help` or
command-specific `--help` for the installed command inventory. The adopted
[CommandKit consumer guide](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/consumers/guide.md)
describes the standard command grammar, inspection, comparison, preview,
validation, smoke tests, and their provider-specific caveats. Help and version
do not load application or model configuration.
## Commands and help
## Processing
```sh
audita process <transcript.json> --glossary <glossary.yml> [flags]
audita config validate --config <config.yml> [flags]
audita config print-effective [flags]
audita version
audita process [flags] transcript.json
```
Root invocation, `--help`, `-h`, and command help succeed without loading a config
file or resolving secrets. Unknown commands and flags return exit code `2`.
`process` accepts exactly one transcript path, either before the flags or after
them. With flags first, put the transcript last. Both config commands reject
positional arguments and process-only flags.
`--config` selects a main configuration and `--overlay` selects its overlay.
`--output` writes the transcript to a file; omission selects stdout. The shared
`--debug` flag enables retained debug artifacts, and `--quiet` suppresses optional
messages. Quiet processing requires an output file or `/dev/null`. See
[operations](operations.md) for output and report lifecycle.
All three commands accept `--config <path>` and `--overlay <name>`. `config validate`
requires an explicit nonempty `--config`; the other commands select the explicit
file, then `/usr/local/etc/audita/config.yml`, then `/etc/audita/config.yml`.
Missing system candidates are skipped; inaccessible selected files are errors.
No file means a configuration error. `AUDITA_CONFIG` is ignored. The selected
overlay is loaded beside the main file, not used to locate profile files.
Explicit process flags override root settings only. `--glossaries` and
`--protected-terms` each accept one JSON array of collection names or `null`;
`[]` clears the selection. Set pipeline order and per-entry overrides in YAML.
Other supplied flags cover profiles, output schema, description, reporting,
chunking, normalization, confidence thresholds, validation budget, and debug
work directory. `audita process --help` lists the available flags. An omitted flag
does not contribute an override. [Configuration](config.md) defines their values
and inheritance.
`audita version` prints `audita <version>` and a newline to stdout, with no
application configuration or provider setup. It returns `0` on success with
empty stderr. `help version`, `version -h`, and `version --help` print command
usage to stdout and return `0`. Help follows standard flag parsing and can
short-circuit trailing arguments. Other invalid invocations return `2` with a
usage error on stderr. The command uses ReleaseKit's public
`buildinfo.Version()` accessor. It reports an injected release identifier when
present, otherwise the executable's main-module version when available and not
`(devel)`, and `dev` when neither source provides a release identifier.
## Inspection and outcomes
## Setting overrides
The application configuration scope uses ConfigKit's native effective values,
source evidence, explanation, comparison, and redaction. PromptKit supplies its
own configuration and resource scopes and request preparation, validation, and
explicit smoke diagnostics. Inspection does not require a processing-ready
pipeline or contact a provider. Smoke tests can contact providers and are run
only on explicit request.
Only flags actually supplied override file or overlay settings. `--modules ''`
clears the module pipeline, and an explicit numeric zero reaches configuration
validation. The shared setting flags are:
| Setting | Flags |
| --- | --- |
| Profile selection | `--proposal-profile`, `--validation-profile` |
| Pipeline and output | `--modules`, `--output-schema` |
| Validation budget | `--validation-max-prompt-tokens` |
| Chunking | `--target-sections`, `--max-section-tokens`, `--min-section-tokens` |
| Normalization (numeric seconds) | `--normalize-max-segment-gap`, `--normalize-ellipsis-gap`, `--normalize-max-segment-duration`, `--normalize-max-segment-tokens` |
| Confidence thresholds | `--glossary-confidence-threshold`, `--homophones-confidence-threshold`, `--spoken-word-confidence-threshold`, `--grammar-confidence-threshold` |
| Context | `--transcript-description` |
| Reporting | `--reporting[=true\|false]`, `--report-dir DIR` |
| Debug | `--debug[=true\|false]`, `--work-dir DIR` |
`process` alone accepts `--glossary` (required) and `--output`.
An omitted or empty output value selects stdout. Nonempty output filenames are
used literally, including whitespace.
Reporting defaults to enabled. `--reporting=false` disables all three ordinary
reports; an explicit false overrides a true file or overlay setting. For stdout
output, reports use the effective `reporting.stdout_dir`, which defaults to the
invocation working directory. For file output, they go beside the transcript
unless `--report-dir` is explicitly supplied. The flag selects the report
directory for either transcript destination; a config-file or overlay value for
`reporting.stdout_dir` alone affects stdout output only. A selected report
directory must be nonblank when reporting is enabled. With reporting disabled,
the report destination is inactive and is not probed. `--debug` separately
enables retained detailed artifacts under `--work-dir`; selecting `--work-dir`
alone does not enable capture. Configuration inspection does not create or
probe these output directories.
Per-role model, base URL, key, timeout, retry, and concurrency flags have been
removed. Models and generation controls belong to PromptKit profiles, backend
credentials and capacity to configuration, and retry/deadline settings to the
shared LLM section. Legacy configuration environment overrides are ignored.
## Streams and exit status
`config validate` loads settings and both selected PromptKit profiles offline,
then prints `config is valid` on success. `config print-effective` accepts
`--view settings|profiles|explain` and defaults to `settings`. Settings writes
ConfigKit's redacted effective-settings YAML. Profiles writes an Audita-owned
YAML summary of selected ID, backend, model, sampling controls, per-attempt
timeout, service tier, and reasoning effort for each role. It omits credentials,
endpoints, and arbitrary extra provider parameters. Explain writes a YAML
sequence of field paths, source origins, and recorded derivations without
resolved setting values. An inherited field may also include an `inheritance`
sequence; each step identifies `from`, `to`, and `reference` paths as JSON
pointers. These fields describe the effective inheritance chain without
including configuration values. A direct child override reports its own source
instead. Debug configuration origins use the same explanation shape. These views
share the same settings and profile validation path as `process`. Unknown views
are usage errors. The effective settings YAML is for inspection and is not
guaranteed reloadable.
For `process`, `--output` writes the corrected transcript to a file and leaves
stdout empty on success. Without it, stdout carries transcript JSON only on
success. Enabled reporting creates `errors.yml`, `recoveries.yml`, and
`report.md` in the selected directory. Successful stderr begins with
`Completed: N corrections applied; E work units with errors; R recoveries.`
Report and debug locations follow only when created. Model-operation failures
can coexist with a completed run and applied corrections; read the YAML reports
for details. Debug artifacts are created only with `--debug`. See the
[report contract](integrations/output-files.md).
Exit codes are `0` for success, `1` for runtime or output failure, and `2` for
usage or configuration input errors. Invalid settings and profiles return `2`;
broken embedded assets, caller cancellation, fatal processing, and required
output failures return `1`. A late report-write failure can follow a published
transcript; exit status and report status remain authoritative.
Configuration commands inspect profiles without contacting a provider.
See the [input files](integrations/transcript-glossary-files.md),
[subprocess contract](integrations/subprocess.md), and
[artifact lifecycle](operations.md) for their respective details.
Process stdout contains transcript JSON when no output file is selected. Ordinary
reports may record recoverable model failures even when processing completes.
Exit status `0` means success, `1` runtime or output failure, `2` invalid usage
or input, and native comparison may return `3` for differences. See the
[output contract](integrations/output-files.md) and [subprocess contract](integrations/subprocess.md).

View File

@@ -1,192 +1,105 @@
# Audita Configuration
This reference describes the version-2 configuration loader in `internal/core/config`.
Use the maintained [local](../examples/minimal-config.yml) and
[service](../examples/production-config.yml) examples with their separately
selected [local profiles](../examples/profiles/local/) and
[service profiles](../examples/profiles/service/), plus the optional
[testing overlay](../examples/overlays/testing.yml).
The [ConfigKit inheritance example](../examples/configkit-inheritance.yml)
shows a local child backend reusing and overriding sibling settings.
Choose real models for your endpoint; a testing profile makes no cost or quality
guarantee. See the [migration guide](migrations/v2.md) for retired inputs.
Audita uses version 3 application configuration. See the [migration guide](migrations/v3.md)
and [examples](../examples/) for complete projects. Command syntax is owned by
[the CLI reference](cli.md).
## Loading and precedence
## Loading and composition
The application selects one main YAML file: an explicit `--config`, then
`/usr/local/etc/audita/config.yml` if present, then `/etc/audita/config.yml`
if present. Missing system candidates are skipped; an explicit missing or empty
path, inaccessible selected file, or absence of any selected file is an error.
`config validate` requires an explicit file. ConfigKit loads built-in defaults,
the selected file, an optional named overlay from `overlays/<name>.yml` beside the
main file, and only explicitly supplied setting overrides, in that order. Imports
and secret-directory bootstrap are disabled. Unknown fields and missing, null,
or non-integer `version: 2` are errors. Version-1 files require migration; no
compatibility loader or implicit environment configuration remains.
An explicit `--config` selects one main YAML file. Without it, ConfigKit searches
the working directory, user configuration directory, and Unix system directories
for `audita` configuration. If no file is found, built-in defaults apply. An
explicit invalid selection or failure of a selected file is an error; search does
not continue after a selected file fails. An omitted version uses the v3 schema;
a supplied version must be `3`.
After overlay and explicit CLI override precedence is applied, ConfigKit resolves
native sibling inheritance before typed decoding. Any input mapping may use
`extends: <sibling-name>`; the child keeps its key, inherits parent settings, and
overrides them with its own settings. Selected overlay and CLI values therefore
participate in inheritance at their normal precedence. The generic syntax and
resolution rules are defined by ConfigKit v0.5.0's
[sibling inheritance contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance).
`extends` is reserved structural input in base files, imports, overlays, and
other application input mappings, including mapping values supplied by explicit
CLI setting overrides. ConfigKit consumes it before typed decoding. A previously
valid backend whose ID is literally `extends` cannot be represented under this
rule. Rename that backend to a non-conflicting ID and update every PromptKit
profile that selects it. This compatibility impact means the upgrade does not
promise that every previously valid configuration remains valid.
Inherited values count as explicit input, including zero, false, empty, and
null. Existing field rules still apply: for example, inherited backend
concurrency must be positive, while an inherited queue capacity of zero is
valid. Parent backends undergo the same decoding and application validation as
other backends, whether or not a selected profile uses them. A child
`credential: null` removes the child's inherited credential; it does not remove
the parent's credential reference, which remains subject to ordinary secret
resolution.
Validation runs after overlays, explicit overrides, and derived settings have
resolved. An override can therefore repair an invalid lower-precedence setting
when that setting can be decoded. ConfigKit retains field origins and derivations
for effective-configuration inspection. Its effective YAML redacts resolved
credentials.
Operator sources may use ConfigKit imports. The selected main file provides the
base for an optional `overlays/<name>.yml`, optional `glossary.yml`, and recursive
`glossary/` YAML files. Without a main source, automatic companion files are not
loaded from the working directory. ConfigKit composes sources, resolves native
inheritance, applies explicit overrides, records derived defaults, and validates
the result. Its [configuration reference](https://forge.maximumdirect.net/base-layer/configkit/src/tag/v1.2.0/docs/config.md)
owns native composition and provenance semantics.
### PromptKit profiles and operator prompts
`profiles_dir` selects a PromptKit configuration root, resolved against the
selected main file's directory unless it is absolute. The root may contain
keyed `profiles`, `backends`, and `prompts` definitions; it is not limited to
profile-only files. IDs come from literal YAML mapping keys, never filenames.
Multiple definitions may share a document, and a document may contribute to any
of the supported mappings. Directory discovery is recursive, but discovery
order does not establish field precedence. Contributions within each source
group must compose without conflicting values.
PromptKit loads its default catalog and Audita's embedded prompts, followed by
optional `prompts.yml` and recursive `prompts/` YAML files beside the application
main. Native imports, resource collections, inheritance, and replacement apply.
Profiles, backends, models, credentials, admission, and provider parameters belong
to PromptKit configuration; see its [format reference](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/formats.md).
The engine captures its sources for an invocation. Edit files and run a new
invocation to pick up changes. Inspection does not require model credentials.
PromptKit assembles sources in this order: its selected catalogs and owned
defaults, Audita's embedded prompt/schema inputs, operator documents under
`profiles_dir`, then the application backend mapping. The application mapping
owns each configured endpoint, concurrency limit, queue capacity, and credential
environment lookup. Other supported provider fields can be contributed by the
operator documents. Imports remain disabled. The current maintained examples
select separate roots: [local](../examples/profiles/local/) and
[service](../examples/profiles/service/), so definitions for one backend do not
enter the other example's final configuration.
## Pipeline and glossary selections
Both selected roles must resolve to an existing profile, registered backend,
nonblank effective model, and nonblank endpoint. The normalized effective
endpoint must match the application backend endpoint. A selected profile cannot
introduce a competing `api_key_env` lookup; the application backend credential
supplies the runtime key when required. Other unselected profiles may be
partial, but every final backend, profile, and prompt definition is validated
when the engine is constructed. A malformed or conflicting unused definition
can therefore make the selected root invalid. A missing selected root is an
operator configuration error; an empty root may contribute no definitions.
`glossary_collections` maps names to collections with an ordered `entries` list.
Each entry has an `id`, `name`, and `category`; `description`, `aliases`, and
`plural` provide optional context. Entries are identified within their collection
by `id`. Root `glossaries` and `protected_terms` select collection names. If root
protection is omitted, it derives from the root glossary selection.
Profile IDs and parent IDs are exact, literal keys. PromptKit profiles can use
native `extends` inheritance; child fields override inherited fields, while
unmentioned controls survive changes to model or backend. For example:
`pipeline.stages` is an ordered list of `module` records; supported modules are
`glossary`, `homophones`, `spoken_word`, and `grammar`. The default sequence is
`glossary`, `homophones`, `glossary`, `spoken_word`, `grammar`. Repeated modules
run separately; `[]` runs normalization without model processing. An optional
nonempty, unique `id` identifies a record in evidence. Each record may override
`glossaries`, `protected_terms`, `confidence_threshold`, and fields under `llm`,
`validation`, and `context`. Omitted fields inherit their resolved root values.
Explicit empty lists and null list selections clear a selection.
```yaml
profiles:
"shared~base":
backend: service
model: example-model
temperature: 0.4
top_p: 0.8
"proposal fast":
extends: "shared~base"
model: faster-model
temperature: 0
```
Presence matters. Explicit `temperature: 0` is sent as zero; omission leaves it
unspecified; null suppresses an inherited temperature. Other fields have their
own native null and omission rules. Provider `extra_params` maps merge
recursively, and a null provider value is sent as JSON null. See the pinned
[PromptKit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
for the complete field semantics.
Operator documents can also override an embedded prompt/version, including its
messages and definition-level output settings. Shared variables supplied by the
adapter are rendered only when an effective message references them; Audita does
not append omitted shared text. Audita's per-call schemas, structural repair
budget, and downstream domain validators remain in force. The engine captures
effective definitions and message-file contents when constructed, so edits are
observed only by a newly constructed runtime.
### Migrating PromptKit profile documents
For the PromptKit v0.11 format, wrap each profile's fields under
`profiles: {<literal-id>: ...}`, remove inner `id`, and replace
`base_profile` with `extends`. Profile and prompt IDs are mapping keys, so they
need not match source filenames. Review zeros, omitted values, and nulls against
the native presence rules before deploying. Select a root whose final
definitions have valid backend references; eager validation applies even to
unselected definitions. See the complete maintained [local](../examples/minimal-config.yml)
and [service](../examples/production-config.yml) examples and the pinned
[consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api).
This incompatible profile-format migration is intended for an Audita major
release; no automatic file conversion is provided.
The [CLI inspection views](cli.md#streams-and-exit-status) show selected
controls and field provenance separately from ConfigKit's redacted settings
YAML. Inspection resolves both roles without provider calls.
Each entry receives its selected context and independent protection. Glossary
correction exempts active recognized spellings from protection for that entry.
A longer exempt name still takes precedence over a shorter protected spelling
inside it. A glossary correction must have at least one selected entry at
processing preflight. Empty definitions and empty protection remain valid
configuration.
## Settings
| Field | Default | Rule |
| Setting | Default | Meaning |
| --- | --- | --- |
| `profiles_dir` | `profiles` | Nonempty PromptKit document root; resolved relative to the selected main file, or absolute. |
| `backends.<id>.endpoint` | None | Required nonempty endpoint for each backend. At least one backend is required. |
| `backends.<id>.concurrency` | `1` | Positive finite capacity; shared by both roles using that backend. |
| `backends.<id>.queue_capacity` | `1024` | Nonnegative; zero disables waiting admission. |
| `backends.<id>.credential` | Unset | Optional environment-variable secret reference; omission or null means unauthenticated use. |
| `llm.proposal_profile` | None | Required nonempty profile ID. |
| `llm.validation_profile` | Final proposal profile | Omission inherits the final proposal selection; explicit empty or null fails. |
| `llm.transient_retries` | `2` | Nonnegative additional attempts per generation. |
| `llm.repair_attempts` | `2` | Additional structural repair generations, from zero through three. |
| `llm.operation_timeout` | `15m` | Positive Go duration string per logical call. |
| `pipeline.modules` | `[glossary, homophones, glossary, spoken_word, grammar]` | Ordered module keys; repeats are meaningful and an explicit empty list runs no correction modules. |
| `output.schema` | `bare-segments` | `bare-segments` or `audita-v1`. See [output formats](integrations/output-files.md). |
| `llm.proposal_profile` | unset | PromptKit profile for proposals; required for processing entries. |
| `llm.validation_profile` | effective proposal profile | Profile for model validation; null clears a selection before fallback. |
| `llm.correction_repair_profile` | effective proposal profile | Profile for correction repair; null clears an inherited selection before fallback. |
| `llm.correction_repair_attempts` | `1` | Nonnegative repair attempts per correction; zero disables correction repair. |
| `llm.transient_retries` | `2` | Additional transient attempts per generation. |
| `llm.repair_attempts` | `2` | Structural repairs, from zero through three. |
| `llm.operation_timeout` | `15m` | Positive duration per logical operation. |
| `validation.max_prompt_tokens` | `2048` | Positive validation batching budget. |
| `chunking.target_sections` | Unset | Optional positive target; omission or null selects automatic chunking. |
| `chunking.max_section_tokens` | `8192` | Positive maximum estimate. |
| `chunking.min_section_tokens` | `2048` | Positive and no greater than maximum. |
| `normalization.max_segment_gap` | `4.0` | Nonnegative numeric seconds. |
| `normalization.ellipsis_gap` | `3.5` | Nonnegative numeric seconds. |
| `normalization.max_segment_duration` | `60.0` | Positive numeric seconds. |
| `normalization.max_segment_tokens` | `2048` | Positive token estimate. |
| `thresholds.glossary`, `.homophones`, `.spoken_word`, `.grammar` | `0.8` each | Confidence thresholds in `[0, 1]`. |
| `context.description` | Empty | Trimmed prompt context, at most 500 bytes. |
| `reporting.enabled` | `true` | Write the ordinary three-file report bundle when processing. |
| `reporting.stdout_dir` | `.` | Report directory for stdout transcript output; a `--report-dir` flag can select reports for either transcript destination. |
| `diagnostics.debug` | `false` | Persist detailed debug artifacts when processing. |
| `diagnostics.work_dir` | `/tmp/audita` | Parent of the retained debug run directory; required only when debug is enabled. |
| `thresholds.<module>` | `0.8` | Root confidence threshold for each module, in `[0, 1]`. |
| `context.description` | empty | Prompt context; at most 500 bytes after trimming. |
| `chunking.target_sections` | unset | Optional positive target. |
| `chunking.max_section_tokens`, `chunking.min_section_tokens` | `8192`, `2048` | Positive section bounds; minimum must not exceed maximum. |
| `normalization.max_segment_gap`, `normalization.ellipsis_gap` | `4`, `3.5` | Nonnegative seconds. |
| `normalization.max_segment_duration`, `normalization.max_segment_tokens` | `60`, `2048` | Positive seconds and token estimate. |
| `output.schema` | `bare-segments` | Output format; see [output contracts](integrations/output-files.md). |
| `reporting.enabled`, `reporting.stdout_dir` | `true`, `.` | Ordinary report capture and default directory for stdout output. |
| `diagnostics.work_dir` | `/tmp/audita` | Parent directory for optional debug capture. |
Module keys select correction modules; validator chains remain built into them.
The loader preserves module order and repetitions. Normalization time settings
accept numeric seconds only. The operation timeout uses a duration string.
`reporting.stdout_dir` must be nonblank when it supplies the active report
destination: for stdout output, or for file output when `--report-dir` is
explicitly supplied. File output without that flag uses the transcript's parent,
so the configured value is inactive. Reporting-disabled runs do not select a
report directory. `diagnostics.work_dir` must be nonblank only with debug
enabled. These path requirements are checked when choosing process destinations,
not during configuration inspection.
For example, an entry can disable correction repair or select another profile:
## Credentials
```yaml
llm:
proposal_profile: local
correction_repair_attempts: 2
pipeline:
stages:
- module: grammar
llm:
correction_repair_profile: review
- module: spoken_word
llm:
correction_repair_attempts: 0
```
A backend credential is a ConfigKit secret reference containing an environment
variable name, such as `credential: AUDITA_SERVICE_KEY`. Every reference that
survives overlay and override resolution must name a set, nonempty variable,
including credentials on backends unused by the selected roles. A replacement
reference resolves only the replacement; null removes the reference. General
`AUDITA_*` and `OPENROUTER_API_KEY` variables have no implicit setting effect.
Secret values are available to the runtime through the resolved reference, while
ConfigKit's effective output and provenance contain no resolved secret values.
Current invocation syntax belongs in [CLI](cli.md).
Processing profiles are selected explicitly; no profile or glossary is selected
automatically. The validation selection falls back after root and per-entry
settings are combined. An explicit root correction repair profile is inherited
by each entry unless overridden or cleared with null; otherwise the entry uses
its effective proposal profile. The repair profile is checked only when its
attempt budget is positive. Each correction repair attempt is a logical call
that can also use the configured transient retries and structural repairs. Null
does not disable validation. Numeric zero and false
retain their field meaning, subject to validation. Inspect recorded origins and
default dependencies using the [native commands](cli.md).

View File

@@ -41,7 +41,9 @@ ownership. Commit discipline and agent Git restrictions are in the
## Baseline Validation
Use Go 1.25.5 or newer. From the repository root:
Use Go 1.27.0 or newer for source builds. The selected release toolchain is
documented in the [release procedure](release.md#prerequisites). From the
repository root:
```sh
go build -o ./bin/audita ./cmd/audita

View File

@@ -1,26 +1,26 @@
# OpenAI-Compatible LLM Integration
Audita uses the pinned [PromptKit](https://gitea.maximumdirect.net/eric/promptkit)
Audita uses the pinned [PromptKit](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09)
built-in OpenAI-compatible transport. PromptKit owns chat-completions wire framing,
provider envelope parsing, transient retries, per-generation timeouts, and JSON
Schema repair. A backend's configured base endpoint must support the selected
profile's model and structured JSON-schema responses; local configuration
inspection does not contact the provider.
Audita selects a role profile, supplies its registered backend credential, and
passes the embedded response schema as a complete per-call output contract. It
prepares each call once, checks that the prepared target still matches the
inspected backend and endpoint, then executes that same prepared request. A
profile cannot introduce a separate credential environment lookup. An absent
backend credential permits an unauthenticated local service.
Audita selects role profiles independently for each pipeline entry and supplies
its domain response schema and per-operation budget. PromptKit owns backend and
credential selection, request preparation, transport, retries, repairs, and
admission. Audita executes the same prepared handle and retains its own domain
validation. Profiles may use native endpoint and credential settings; local
inspection does not resolve a credential value or call the provider.
`llm.transient_retries` counts additional attempts for eligible transient
failures; explicit zero makes one attempt. `llm.repair_attempts` counts
additional generations for schema-invalid model output. The operation timeout
covers preparation, capacity waiting, retries, generation, and repair. Caller
cancellation or an earlier deadline also applies. See [configuration](../config.md#settings)
for defaults and [PromptKit v0.12.0's consumer contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md)
for the pinned protocol and retry details.
for defaults and the [PromptKit consumer contract](https://forge.maximumdirect.net/base-layer/promptkit/src/tag/v0.21.0/docs/consumers/pkg-promptkit.md)
for protocol and retry details.
A model operation that exhausts structural repair, a live-parent local operation
timeout, capacity rejection, transport failure, malformed provider response,
@@ -31,4 +31,4 @@ continue. Caller cancellation, broken assets or configuration, and unknown
internal failures remain fatal and cancel peers. Audita decodes into domain
types only after PromptKit validates the JSON. Ordinary reports use safe reason
codes and counts rather than provider messages. Full request/response content
is written only with `--debug`; see [operations](../operations.md#diagnostics-directory).
is written only with `--debug`; see [operations](../operations.md#debug-evidence).

View File

@@ -5,7 +5,7 @@
The corrected transcript uses the selected `output.schema`: `bare-segments`
or `audita-v1`. Its schema and text rules are unchanged by resilience
reporting. See [configuration](../config.md#settings) for selection and
[operations](../operations.md#required-output-files) for placement and writes.
[operations](../operations.md#outputs-and-lifecycle) for placement and writes.
With reporting enabled, Audita produces three ordinary files:
@@ -46,9 +46,9 @@ remain in opt-in debug files.
## Error and Recovery Records
An `errors` record with `scope: work_unit` identifies zero-based `stage_index`
and `chunk_index`, module key/instance, initial segment range, applied-correction
and `chunk_index`, optional `stage_id`, module key/instance, initial segment range, applied-correction
count, and a `failures` sequence. Each failure identifies the operation
(`proposal`, `validation`, or `validation_input`), optional validator and
(`proposal`, `validation`, `validation_input`, or `correction_repair`), optional validator and
zero-based batch index, safe reason/message, sorted proposal indices when
known, and execution counts when available. Multiple failed batches in one
work unit form one error record. `scope: run` instead records phase, safe reason
@@ -59,11 +59,15 @@ For `validation_decisions_invalid`, proposal indices identify only candidates
whose required decisions were unavailable. Other candidates in the same batch
may still be approved or rejected normally.
Each recovery identifies its work unit and successful operation, with sorted
Each recovery identifies its work unit, optional `stage_id`, and successful operation, with sorted
`reasons`, a safe message, available execution counts, and sorted
`filtered_proposals` details. One recovery can contain `output_repaired`,
`service_recovered`, and `proposals_filtered` together. Filtering covers
defective response proposals; low confidence, protected-term rejection,
`service_recovered`, `correction_revised`, and `proposals_filtered` together.
`correction_revised` means a repair yielded a fully validated revision; its
application outcome is recorded in the Markdown report. Filtering covers
defective proposals and revisions under their originating operation. Only
original proposal operations count toward `proposal_responses_with_filtering`.
Low confidence, protected-term rejection,
ordinary editorial rejection, and an apply-time conflict caused by an earlier
accepted correction are not filtering recoveries. Failed operations are errors,
not recoveries. The initial chunk identity remains stable even if earlier
@@ -79,23 +83,29 @@ is an error even when unexpected indices were also ignored.
`report.md` includes run ID, status, corrections, work coverage, error and
recovery totals, input/output paths, and full work-unit error and recovery
matrices. Rows show every initial chunk with 1-based chunk number and initial
segment range; columns follow configured module-instance order, including
repeats. Completed error cells are `0` or `1`; recovery cells are operation
segment range; columns follow configured pipeline order, including repeated modules. Labels show
the one-based position and module, plus the operator ID when supplied. Completed error cells are `0` or `1`; recovery cells are operation
counts. `—` means not started and `n*` means interrupted with n facts recorded.
An empty stage or chunk set is identified as having no work units. Short reason breakdowns count
records containing each reason; a recovery can have several reasons, so those
counts need not sum to the recovery total. The report refers to both YAML
files. It does not include transcript excerpts or correction lists.
counts need not sum to the recovery total. When a repair was attempted, a
Correction repair table lists its stage, chunk,
original proposal index, semantic attempt count, and final outcome (`applied`,
`approved_skipped`, `withdrawn`, `rejected`, `exhausted`, `unavailable`, or
`interrupted`). The report refers to both YAML files. It does not include
transcript excerpts or correction lists.
## Debug-Only Files
With `--debug`, `correction-ledger.json` contains proposed candidates and
their application, rejection, or interruption details. `llm-execution.json`
holds logical calls and PromptKit execution events; `utilization-diagnostics.json`
groups backend capacity and timing observations while preserving known and
complete usage semantics. Interaction request/response artifacts, selected
profiles, configuration provenance, and input snapshots are also debug-only.
holds logical calls and PromptKit execution events. Interaction request/response
artifacts and input snapshots are also debug-only. Records carry zero-based
`stage_index` and optional `stage_id` where an entry is identified.
These files are diagnostic evidence, not an alternate ordinary report schema.
For a repaired proposal, one ledger row preserves the original proposed edit,
the final applied segment and span when applicable, and repair attempts with
revision decisions and interaction artifact paths.
Each candidate-backed ledger entry includes `proposed_corrected_text`, the
literal replacement span, even when it is `""` for a deletion. Applied entries
also include `applied_corrected_text`, likewise `""` for an applied deletion.

View File

@@ -6,15 +6,15 @@ The [CLI reference](../cli.md) owns command syntax, streams, and exit semantics;
## Invocation
Use an argument array with the transcript path and required glossary argument,
following the [process examples](../cli.md#commands-and-help). For machine workflows,
Use an argument array with the transcript path and configured glossary selections,
following the [process examples](../cli.md#processing). For machine workflows,
select an explicit transcript file destination and a configuration path. Reports
go beside the transcript unless an explicit `--report-dir` selects another
directory. For stdout output, reports use the effective `reporting.stdout_dir`
(defaulting to the invocation working directory), including a `--report-dir`
override. A config-file or overlay value for `reporting.stdout_dir` alone does
not redirect file-output reports. `--work-dir` selects a debug root, while
configuration or `--debug` enables capture; the path flag alone does not.
`--debug` enables capture; the path flag alone does not.
Supplying a configuration path also makes file selection predictable; only
explicitly referenced credential variables are read from the environment.
@@ -26,19 +26,20 @@ or cancellation in the parent process when needed.
## Result Handling
Interpret the exit result according to the
[CLI output and exit contract](../cli.md#streams-and-exit-status).
[CLI output and exit contract](../cli.md#inspection-and-outcomes).
Consume transcript/report data through the
[output-file contract](output-files.md), and treat stderr as human-readable detail.
The three reports may be absent after early configuration or destination
failures. After initialization, failed reports can describe invalid input or
partial processing and use the same selected report directory as successful
runs. Exit `0` can include recoverable model errors and applied
Audita loads configuration and parses the input transcript before report
initialization. Early destination failures can also leave no reports. After
initialization, failed reports can describe partial processing or output
failures and use the same report directory as successful runs.
Exit `0` can include recoverable model errors and applied
corrections; inspect `errors.yml` and `recoveries.yml` for those outcomes. A
transcript may have been written before a later required report failure, so
the file alone does not prove completion.
If processing fails, preserve available evidence and follow
[operations recovery](../operations.md#failure-inspection-and-recovery).
[operations recovery](../operations.md#recovery).
Ordinary reports are independent of optional retained debug evidence. Parent
cancellation reaches the child through its context; Audita joins workers and
attempts failed-status report finalization before exiting. Uncatchable
@@ -49,4 +50,4 @@ file nor a debug directory alone establishes successful completion.
- [Transcript and glossary inputs](transcript-glossary-files.md)
- [LLM endpoint compatibility](openai-compatible-llm.md)
- [Configuration inspection commands](../cli.md#commands-and-help)
- [Configuration inspection commands](../cli.md#inspection-and-outcomes)

View File

@@ -28,23 +28,11 @@ The [tiny transcript](../../examples/tiny-transcript.json) is a complete copyabl
input. Invalid JSON, unsupported top-level shapes, empty transcripts, invalid field
types, duplicate supplied IDs, and the validation failures above are rejected.
## Glossary YAML Contract
## Glossary collections
The top-level object contains a `glossary` list with at least one entry.
| Entry field | Type and validation |
| --- | --- |
| `name` | Non-empty string |
| `category` | Non-empty string |
| `summary` | Non-empty string |
| `aliases` | Optional list of non-empty strings |
| `plural` | Optional string |
The parser checks empty strings without trimming them. Unknown YAML fields are
ignored by the glossary parser, unlike the stricter
[configuration-file schema](../config.md#loading-and-precedence).
The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input.
Malformed YAML, an empty/missing glossary, invalid shapes, missing required text,
and empty aliases are rejected. These formats describe input acceptance, not
additional vocabulary restrictions.
Glossary definitions are composed as application configuration, including the
optional companion inputs described by the [configuration reference](../config.md).
The [local example](../../examples/local/glossary.yml) provides a complete
collection. Configuration may contain empty collections; processing preflight
requires every glossary-correction entry to select at least one glossary entry.
The previous standalone `glossary` file format is not a processing input.

View File

@@ -9,7 +9,7 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
1. Update fields, defaults, loading, and validation in `internal/core/config`.
2. Update file, selected overlay, credential reference, and CLI mapping as
applicable. Preserve the precedence defined in the
[configuration contract](../config.md#loading-and-precedence), including the
[configuration contract](../config.md#loading-and-composition), including the
distinct validation and effective-config commands.
3. Update redaction when secrets are involved.
4. Review configuration tests and related CLI tests for precedence, validation,
@@ -29,33 +29,18 @@ CLI wiring, and maintained examples. Definitions of behavior belong to the
The `version` route uses ReleaseKit's public `buildinfo.Version()` accessor and
stays independent of application runtime assembly. Preserve its behavior from
the [CLI contract](../cli.md#commands-and-help) when changing the command; the
the [CLI contract](../cli.md) when changing the command; the
adoption decision is recorded in [ADR 0001](../adr/0001-releasekit.md).
## Validating Examples
Keep complete copyable files in [examples](../../examples/) and link to them from
their canonical contract documentation. Keep them valid and secret-free.
From the repository root, validate the maintained configuration examples:
```sh
go run ./cmd/audita config validate --config examples/minimal-config.yml
go run ./cmd/audita config validate --config examples/configkit-inheritance.yml
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml
AUDITA_SERVICE_KEY=synthetic-example go run ./cmd/audita config validate --config examples/production-config.yml --overlay testing
go run ./cmd/audita config print-effective --config examples/minimal-config.yml --view profiles
go run ./cmd/audita config print-effective --config examples/configkit-inheritance.yml --view explain
```
The local and ConfigKit inheritance files select `profiles/local`; the service
file selects `profiles/service`, including the service testing profiles. Keep
these roots separate because PromptKit validates every final definition in a
selected root, including definitions that are not selected for a role.
Only referenced credentials read environment values; use a controlled synthetic
value when validating the service example. Check affected CLI examples against the
parser and use schema tests for maintained transcript/glossary inputs. Example
validation should not invoke a live LLM merely to check documentation.
Run the commands in the [example guide](../../examples/README.md) from each
project's directory. `internal/cli/examples_integration_test.go` exercises the
maintained projects with a controlled provider, and the preview fixture can be
prepared without inference. Review automatic companion sources and overlays in
their actual project directories. Keep service credentials synthetic in tests.
## Runtime Change Checks

View File

@@ -24,36 +24,29 @@ retain their execution counts and never produce a recovery record.
`internal/core/diagnostics.OutputSession` owns active destination checks,
transcript preflight, report initialization, regular-file replacement, and direct
stream publication. Transcript input is protected from early report writes but
may be the final transcript destination. The LLM runtime records successful reads
through PromptKit's profile filesystem and supplies those source paths to CLI;
may be the final transcript destination. The LLM runtime records consumed PromptKit sources and effective host message
files and supplies those source paths to CLI;
no second filename-based discovery rule is used.
CLI composes it after configuration/profile inspection and before reading
inputs or dispatching model work. It does not create ordinary report parents.
CLI composes it after configuration, transcript acquisition, and preflight
readiness and before dispatching model work. It does not create ordinary report parents.
On a final write error it attempts other enabled reports and one failed-status
pass; a published transcript is never erased by a later report failure.
Debug is independent. Only `diagnostics.debug` creates a unique retained child
Debug is independent. The shared `--debug` flag enables a unique retained child
under `diagnostics.work_dir`. CLI passes empty diagnostics paths and nil writers
when debug is off. With debug on, source/normalized snapshots, invocation,
effective settings, origins, profiles, initial chunk summary, correction ledger,
indexed execution/utilization, and model interactions are written. Runner
candidate capture includes proposals later rejected or interrupted. Ledger
mapping uses optional string pointers so an empty proposed replacement is
serialized for every candidate and an empty applied replacement appears only
when that candidate applied; see the [public artifact contract](../integrations/output-files.md#debug-only-files).
Debug write failures reach the CLI through an explicit concurrency-safe diagnostic
callback and produce redacted stderr messages without aborting processing.
Completed validator decisions returned with a fatal batch error remain available
to the debug ledger; they never authorize applying an incompletely validated edit.
Runner decision records carry the validator's explicit approved, rejected, or
unavailable disposition into the ledger. Unavailable candidates have a distinct
top-level ledger disposition, while fatal interruptions remain interrupted when
no rejection or unavailable decision was returned. Successful validation
interaction payloads contain the full decoded response, final content, and
reconciliation outcomes; ignored indices do not become ledger candidates.
Every debug writer uses all configured backend secrets for redaction, including
unselected backend credentials and source snapshots. Debug can still contain
sensitive transcript and prompt content.
when debug is off. With debug enabled, it captures source and normalized
snapshots, invocation, effective settings, correction ledger, LLM execution
observations, and model interactions. Candidate capture includes proposals later
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable; their retained content
is redacted using the selected role credentials. Ledger mapping preserves empty proposed and applied
replacements and distinguishes unavailable candidates from interrupted work.
A concurrency-safe diagnostic callback reports optional write failures without
aborting processing. Diagnostic redaction uses credential source metadata; debug
artifacts can still contain sensitive transcript and prompt content. Execution
position and optional operator ID are carried separately from generated safe
artifact path coordinates.
## Key Tests

View File

@@ -1,60 +1,30 @@
# Audita LLM Runtime
## Client Boundary
`llm.NewRuntime` assembles one captured PromptKit engine using its default
catalog, Audita's embedded prompts and schemas, and optional operator sources
anchored to the recorded application main context. Captured host sources and
message files are passed to output collision checks. An embedded-only diagnostic
construction distinguishes shipped asset failures from operator-source failures;
it never replaces the requested engine.
`llm.NewRuntime` assembles one PromptKit engine for both roles. PromptKit's
catalogs and defaults are followed by Audita's embedded prompt and schema
inputs, operator documents from `profiles_dir`, and the highest-priority
application backend mapping. Audita owns backend endpoints, capacity settings,
and resolved credentials; native PromptKit layering owns profile inheritance,
prompt overrides, and provider-specific fields. Imports are disabled. The
runtime inspects both selected roles offline, requiring complete model and
endpoint values and rejecting competing profile credential lookups. A
role-bound `llm.RoleClient` implements `structuredoutput.Client`; proposal and
validator contracts alias that interface. Both roles share engine-local backend
capacity.
Each resolved pipeline entry receives role-bound clients with its selected
profiles and effective budgets. Clients share engine-local backend admission.
The adapter uses `PrepareExecution` and `RunPrepared` on the same handle, passes
Audita's per-call schema and repair budget, and decodes only validated results.
PromptKit owns backend and credential selection. The adapter uses captured
credential source names for artifact redaction without inspecting unrelated
environment values. See the [integration contract](../integrations/openai-compatible-llm.md)
for failure behavior and the [configuration reference](../config.md) for sources.
Successful construction captures the final definitions and referenced message
contents. The runtime does not reload operator documents or message files during
calls. If final engine construction fails, an embedded-only diagnostic
construction distinguishes shipped asset failures from invalid operator input.
An asset failure remains an asset error; when embedded construction succeeds,
the original failure remains a configuration error. The diagnostic engine
never replaces or executes the requested configuration, and caller cancellation
skips this path. The captured inspection also supplies an internal source
ledger: all loaded host YAML plus effective, source-relative host
`content_file` resources are tracked for output-collision checks.
When correction repair is enabled, the proposal call retains its prompt,
effective variables, prepared messages, and full final response for section-local
replay. The adapter renders the repair definition with the selected profile,
appends only that branch’s history, and checks the original prepared prefix
before execution. The complete conversation is sent to the selected provider;
provider rejection ends that branch as unavailable. The requested repair
definition is retained in call attribution alongside native base-prompt metadata.
For each logical call, the adapter starts an operation context and a diagnostic
record before preparation. It supplies the reserved hardening text, the selected
profile and resolved backend credential, and the complete embedded schema/repair
contract. `PrepareExecution` produces the exact messages and hashes retained in
an Audita snapshot. The adapter checks the prepared backend and endpoint before
`RunPrepared`. It records available final content, usage, model, and profile
metadata and decodes only a validated result. Call hooks expose logical
start/finish records and PromptKit execution events for debug utilization.
The [integration contract](../integrations/openai-compatible-llm.md) describes
wire ownership and failure behavior. `structuredoutput.Failure` has a safe
message, fixed reason code, typed cause, and recoverable/fatal/canceled class.
Counts distinguish provider attempts, transient retries, and structural
repairs, including on errors. Secrets from all configured backends, including unselected
ones, are available to the diagnostics redaction boundary.
CLI explain output and debug configuration origins share a value-free projection
of effective field origins. The [CLI contract](../cli.md) defines its optional
inheritance evidence fields.
## Orchestration
CLI processing supplies two role clients from the same engine to the runner.
PromptKit's per-backend admission applies to both roles and every generation,
retry, and repair. The runner still processes module instances in order,
collects concurrent section work deterministically, and cancels remaining
section work only on fatal failures. `llm.DiagnosticsWriter` remains the protected
request/response artifact writer. `llm.ObservationCollector` captures metadata
events under a short mutex and freezes them after runner work joins. Its logical
records include preparation failures; summary usage comes from terminal events
once per call and retains knownness and completeness for each field. Runner wall
and validator timings remain separate from provider attempts. Ordinary reports
derive compact runner outcomes; full observations are debug-only.
The observation collector tracks logical calls and execution events. Ordinary
reports use compact safe outcomes; request and response evidence is optional
debug material. The runner preserves ordered pipeline traversal and deterministic
section collection while canceling and joining peers on fatal errors.

View File

@@ -13,6 +13,9 @@ Modules implement `contracts.TranscriptModule`:
- `Validators()`
- `Propose(ctx, req)`
Built-in modules also expose `RepairPromptID()` for the runner to select their
repair definitions; proposal-only callers retain the existing interface.
Runner resolves configured module specs to module instances through `internal/framework/modules`.
## Built-In Modules
@@ -37,9 +40,9 @@ Shared proposal-generation plumbing is centralized in:
Module packages own:
- prompt selection (`internal/prompts` prompt IDs).
`proposal_generation.BuildProposalVariables` encodes the glossary, selected
transcript section, and trimmed description once for all four modules. PromptKit
renders their embedded definitions from those variables and validates the
Each stage prepares the encoded glossary context and protected spellings once.
Proposal generation adds the selected transcript section and trimmed description
for each request. PromptKit renders the selected definition and validates the
correction-set schema before Audita maps proposals. Domain validators still
check confidence, applicability, protected glossary terms, and replacement
policy. Model-facing segment IDs are one-based positions within the proposal
@@ -61,6 +64,10 @@ Module packages call the built-in chain resolver at construction.
## Failure and Outcome Behavior
Each actionable rejection can be revised or withdrawn within its original
section. A revision retains the original proposal identity and must pass the
full validator chain before application.
Module construction failures are fatal. Recoverable proposal model failures
record an unsuccessful operation and yield no candidates for that section;
other sections and module instances continue. Fatal adapter or application
@@ -88,4 +95,4 @@ outcomes are defined by the [report contract](../integrations/output-files.md).
- `internal/modules/*/module_test.go`
- `internal/framework/modules/registry_test.go`
- `internal/framework/proposal_generation/*_test.go`
- `internal/cli/run_test.go` (pipeline/report integration)
- `internal/cli/process_ordered_integration_test.go` (pipeline/report integration)

View File

@@ -12,7 +12,7 @@ selectable values and defaults belong to
Configuration validation checks registry support. CLI orchestration resolves the
selected encoder, passes the final transcript to it, and routes the encoded bytes
to the destination defined by the [CLI contract](../cli.md#streams-and-exit-status).
to the destination defined by the [CLI contract](../cli.md#inspection-and-outcomes).
An unsupported key fails validation or resolution rather than selecting an
implicit fallback.

View File

@@ -14,6 +14,7 @@ Executable and command orchestration:
Core deterministic components:
- `internal/core/config`
- `internal/core/glossary`
- `internal/core/schema`
- `internal/core/normalization`
- `internal/core/chunking`
@@ -24,6 +25,7 @@ Core deterministic components:
Framework orchestration and contracts:
- `internal/framework/contracts`
- `internal/framework/preflight`
- `internal/framework/modules`
- `internal/framework/proposal_generation`
- `internal/framework/proposals`
@@ -66,7 +68,7 @@ artifact lifecycle.
## Component Responsibilities
Core components implement configuration defaults/loading/validation/redaction,
transcript and glossary schemas, normalization and chunking, output encoding,
transcript schemas and derived glossary views, normalization and chunking, output encoding,
diagnostics artifact helpers, and public report structures.
Framework components implement module and validator contracts, proposal

View File

@@ -4,8 +4,9 @@ The [input contract](../integrations/transcript-glossary-files.md) owns input
formats; [configuration](../config.md) owns settings; the
[output contract](../integrations/output-files.md) owns serialized results.
1. CLI filesystem adapters read transcript and glossary files. Schema parsing
produces source objects; transcript segments enter canonical normalization.
1. CLI loads resolved configuration and transcript input. Preflight checks all
selected glossary views and roles before the output session prepares resources.
Transcript segments then enter canonical normalization.
2. Normalization sorts and merges segments using configured time and token
limits, reassigning IDs where needed.
3. The runner partitions the normalized transcript exactly once before its
@@ -18,10 +19,14 @@ formats; [configuration](../config.md) owns settings; the
may preserve a checked subset of completed decisions as diagnostic evidence.
Recoverable failed batches mark their candidates unavailable; successful
survivors continue through all required validators.
6. Only fully approved proposals reach application. Corrections apply in
6. Actionable rejections can enter independent section-local repair branches.
Each revision is mapped to the original section and revalidated from the
start. Withdrawal, exhaustion, and recoverable failures finish the branch
without affecting independent approvals.
7. Only fully approved proposals reach application. Corrections apply in
proposal-index order within their owning section. Existing replacement
policies and apply-time safety checks resolve stale conflicts.
7. CLI encodes the final transcript and publishes active files through the
8. CLI encodes the final transcript and publishes active files through the
output session. `framework/processreport` builds one snapshot for YAML,
Markdown, and CLI totals. Debug evidence is captured separately when enabled.

View File

@@ -1,37 +1,43 @@
# Audita Prompt Registry
`internal/prompts` embeds seven PromptKit YAML definitions under
`assets/prompts`. The IDs and selected version remain:
`internal/prompts` embeds eleven PromptKit definitions in the `audita`
collection. The four proposal IDs are `audita/modules.glossary.proposal`,
`audita/modules.homophones.proposal`, `audita/modules.spoken_word.proposal`,
and `audita/modules.grammar.proposal`; the corresponding repair IDs replace
`.proposal` with `.repair`. The three validator IDs are
`audita/validators.spoken_form_plausibility`,
`audita/validators.meaning_reversal_review`, and
`audita/validators.editorial_review`.
- `modules.glossary.proposal` (`v1`)
- `modules.homophones.proposal` (`v1`)
- `modules.spoken_word.proposal` (`v1`)
- `modules.grammar.proposal` (`v1`)
- `validators.spoken_form_plausibility` (`v1`)
- `validators.meaning_reversal_review` (`v1`)
- `validators.editorial_review` (`v1`)
PromptKit composes embedded defaults with operator prompt sources. Shared
instructions come from the embedded fragments in `assets/shared`; the request
shared-variable helper fills absent `HardeningPolicy`, `ProposalContract`,
`RepairContract`, and `ValidationContract` variables. The processing adapter
supplies the embedded instruction fragments for those names. Processing supplies
Audita's response-schema contract separately.
The operator's prompt definition controls which variables its template uses.
Proposal focus and validator acceptance intentionally differ: minor editorial
cleanup may be accepted even when the proposing module was not asked to perform
it. Follow the [architecture boundary](../policy/architecture.md#correction-and-concurrency-invariants)
when maintaining proposal, validation, and repair instructions; the difference
alone is not a reason to tighten validation. The testing rationale is recorded in
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md).
For a standard request preview or validation, select a native PromptKit profile
and supply the template variables used by the selected prompt. The shipped
proposal definitions use `TranscriptDescriptionBlock`, `GlossaryJSON`,
`ProtectedTermsJSON`, and `SectionJSON`. Glossary entries provide context,
and the separate spelling list defines protection. Preview defaults the
spelling list to `[]` when omitted. The validator definitions use `TranscriptDescriptionBlock` and
`PayloadJSON`. The latter is supplied candidate JSON and does not require a
proposal inference call. Repair definitions use `RepairContract` and
`RepairRequestJSON`, containing the original proposal reference, current
correction, and ordered findings. An operator replacement may use different
variables.
See the [upstream CLI reference](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/docs/cli.md)
for the standard preview, validation, and smoke commands.
PromptKit reads the embedded definitions as shipped defaults and renders their
messages from named string variables. Operator documents in `profiles_dir` may
contribute native keyed prompt definitions and override an embedded
prompt/version, including its messages and definition-level output settings.
The effective operator text is captured when the runtime is constructed; the
embedded assets remain the source of Audita's shipped task instructions. The
role-bound adapter supplies
`HardeningPolicy` alongside `ProposalContract` and `ValidationContract` from
embedded fragments under `assets/shared`, after caller variables so callers
cannot replace those instructions. Common literal-span and identity guidance
lives in the proposal fragment; shared candidate-reference, context, and
confidence guidance lives in the validation fragment. Module-specific policies
and examples remain in YAML. Proposal prompts receive the section JSON, glossary
JSON where used, and transcript description block. Validator prompts receive
the correction payload and description block.
The embedded prompt assets retain the shipped task instructions and
output-schema association. An operator override controls which template
variables its messages reference: supplying a shared variable does not append
its text when the effective message omits that reference. The role-bound runtime
supplies the complete JSON Schema output contract and repair count; the
definition does not own an operator-facing repair default.
The correction-set schema requires a string replacement and accepts an empty
string. Proposal guidance uses it for eligible span deletion and permits literal
spacing edits; validator guidance evaluates the full before/after segment under
@@ -39,14 +45,14 @@ each validator's existing remit. A blank resulting segment remains a domain
rejection, not a structural schema error.
`internal/framework/responseschema` remains the only embedded source for
`correction_set.v1.json` and `validator_decision_set.v1.json`. It exposes that
`correction_set.v1.json`, `correction_repair.v1.json`, and
`validator_decision_set.v1.json`. It exposes that
filesystem to PromptKit and retains Audita's schema-key lookup. PromptKit's
prepared execution supplies definition and rendered hashes; Audita does not
recompute or promise equality with hashes from the removed template renderer.
The small Audita registry records only prompt ID, version, source, and embedded
path. Prepared runtime metadata belongs in diagnostics rather than the registry.
The small Audita registry records only prompt ID, source, and embedded path. Prepared runtime metadata belongs in diagnostics rather than the registry.
The structured-call contract carries prompt ID/version, string variables,
The structured-call contract carries prompt ID, string variables,
response-schema key, and logical-call attribution. Module and validator callers
pass these values through the role-bound adapter. See the
[LLM runtime](llm-runtime.md) for runtime assembly and

View File

@@ -46,6 +46,14 @@ For each module section:
3. record decisions and compact operation outcomes;
4. carry only approved proposals forward.
When correction repair is enabled, the runner also gathers independent findings
for rejected candidates when a validator has usable evidence. Unavailable
assessments remain unavailable, and only candidates that pass every applicable
check can advance. A known rejection can enter repair even when another
assessment is unavailable; an unavailable assessment alone does not. Revised
candidates run the applicable checks again. A zero repair budget retains the
original shortlist flow.
Decision cardinality is enforced inside Audita: each candidate proposal must
receive exactly one outcome per validator it reaches. The runner still rejects
duplicate, missing, or unknown identities in an internal validator result.
@@ -116,19 +124,21 @@ is covered by [report assembly](diagnostics-reporting.md).
- `internal/validators/*_test.go`
- `internal/framework/validators/*_test.go`
- `internal/framework/processreport/correction_ledger_test.go`
- `internal/cli/run_test.go`
- `internal/cli/process_ordered_integration_test.go`
## Supplemental Validation Context
Request construction prepares one segment-position lookup and protected vocabulary
for all candidates. Glossary selection remains candidate-specific, including the
proposed replacement text.
Each stage prepares a context matcher and indexes its selected glossary entries
by identity. Request construction uses a segment-position lookup and returns
matched entries in selection order for each candidate, including matches in the
replacement text. The deterministic guard and application check share the
stage's separate protection matcher.
Each candidate includes the immediately preceding and following source segments
available within its fixed chunk, labeled `preceding` or `following` with speaker,
text, and categories, plus glossary entries whose names, aliases, or
recognized plurals appear in its original/resulting text or those neighbors.
Glossary matching reuses the protected vocabulary; entries retain glossary order.
Glossary matching uses the same recognition rules; entries retain selection order.
Model-facing candidate and neighboring context omit internal segment IDs.
This supplies evidence such as a troupe definition for a proposed `Jesters`
correction or an adjacent speaker turn explaining deliberate repetition. A correction
@@ -140,7 +150,8 @@ payload token estimate. If one enriched item exceeds the configured limit, optio
neighbors are removed from the end first, then glossary entries from the end,
until it fits or only required candidate data remains. Required before/after text
is never truncated. Existing oversized-candidate handling still applies if that
required data does not fit. Context is supplemental, not a promise that every
required data does not fit. The built-in estimator reuses encoded fragment
counts for fitting and batching. Context is supplemental, not a promise that every
relevant definition or neighboring utterance always fits. This uses the existing
payload estimator, not a new guarantee about the provider's full context window.

View File

@@ -2,8 +2,8 @@
Audita v2 adopts ConfigKit for configuration and PromptKit for LLM execution.
Version-1 configuration files, environment overlays, and retired flags are not
accepted. Copy a maintained [local](../../examples/minimal-config.yml) or
[service](../../examples/production-config.yml) example, then select real models
accepted. Copy a maintained [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/minimal-config.yml) or
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/examples/production-config.yml) example, then select real models
in the accompanying `examples/profiles/` files. A testing overlay is shown at
`examples/overlays/testing.yml`; its model choice carries no cost, speed, or
quality guarantee.
@@ -122,7 +122,7 @@ capacity and usage fields retain known/complete flags. Debug runs write
`effective-config.redacted.yaml`, `selected-profiles.yaml`,
`config-origins.yaml`, `utilization-diagnostics.json`, and
`llm-execution.json`. Request and response artifacts remain redacted debugging
material. See [operations](../operations.md#diagnostics-directory) for placement
material. See [operations](../operations.md#debug-evidence) for placement
and paths, [configuration](../config.md) for all settings, and the
[CLI reference](../cli.md) for current flags.

27
docs/migrations/v3.md Normal file
View File

@@ -0,0 +1,27 @@
# Migrating to application configuration v3
Audita 3.1.x uses application configuration schema v2. For the cutover to schema
v3, set `version: 3` in the application file and move glossary entries into
configuration. Use the [configuration reference](../config.md) and
[maintained examples](../../examples/) for the new formats.
- Replace `pipeline.modules` with ordered `pipeline.stages` records containing
`module` and any per-entry overrides. An explicit empty list runs normalization.
- Define named `glossary_collections` in configuration or its companion imports;
select them using `glossaries` and `protected_terms`. Rename each legacy
entry's `summary` to `description` to preserve its prompt context. Remove
`--glossary` and `--modules` from invocations.
- Move profiles and backends out of the application configuration and retired
`profiles_dir` layout into PromptKit resources. Use its current list records,
IDs, credential sources, and backend admission settings. Select proposal and
validation profiles in application `llm` settings.
- Use the shared `--debug` flag for capture; `diagnostics.work_dir` remains the
parent directory. Review report and output destinations before running.
- Completion counts are hidden by default. Use `--verbose` to see them on
stderr, including when ordinary reports are disabled. `--debug` also shows
counts unless followed by `--quiet`. See [CLI reporting](../cli.md#processing).
Inspect configuration and resources before processing, then try a controlled
small transcript. Processing preflight checks every glossary correction and
selected role before preparing outputs or invoking a model. See [operations](../operations.md)
for lifecycle and recovery.

View File

@@ -1,94 +1,66 @@
# Audita Operations
This guide owns physical output placement and failure handling. The
[CLI reference](cli.md) owns flags and streams, [configuration](config.md) owns
settings, and the [output contract](integrations/output-files.md) owns schemas.
This guide owns output placement and failure handling. The [CLI reference](cli.md)
owns invocation, [configuration](config.md) owns settings, and the
[output contract](integrations/output-files.md) owns serialized formats.
## Normal Workflow
## Processing
1. Validate a version-2 configuration and its selected profiles with
`audita config validate --config config.yml`.
2. Prepare a transcript and glossary following the [input contract](integrations/transcript-glossary-files.md).
3. Run `audita process input.json --config config.yml --glossary glossary.yaml --output corrected.json`.
4. Check the exit status, stderr summary, corrected transcript, and reports.
A completed run can contain recoverable model errors and still apply
corrections that passed every required validator.
Inspect the application and PromptKit resources, prepare a transcript, then run
`audita process` with the desired destination. The [examples](../examples/README.md)
show complete projects. Processing preflight checks every configured glossary
correction and selected profile before output preparation and model work. Empty
protection and an empty pipeline are valid. A completed run can contain
recoverable model errors while still applying independently validated edits.
Correction repair sends the complete original proposal exchange and branch
history to the selected repair provider. A provider rejection ends that branch
as unavailable; other approved edits can still be applied. See
[configuration](config.md#settings) for the attempt budget and profile selection.
## Required Output Files
## Outputs and lifecycle
Reporting is enabled by default. For file transcript output, `errors.yml`,
`recoveries.yml`, and `report.md` go in the transcript output's parent directory
unless `--report-dir` is explicitly supplied. For stdout transcript output, they
go in the effective `reporting.stdout_dir` (default `.`), including a
`--report-dir` override. A config-file or overlay value for
`reporting.stdout_dir` alone does not redirect file-output reports. An active
selected report directory must be nonblank. `--reporting=false` suppresses all
three reports and leaves their destination inactive, so it is not probed merely
because `--report-dir` was supplied. It does not suppress the stderr summary or
fatal errors. Stdout contains only transcript JSON; file output leaves it empty.
Reporting is enabled by default. For transcript file output, `errors.yml`,
`recoveries.yml`, and `report.md` are placed beside that file unless `--report-dir`
is supplied. For stdout transcript output they use the effective report directory.
Disabling reporting leaves its destination inactive and unprobed. Quiet mode
requires a transcript file or `/dev/null`; it suppresses optional messages, not
failures. Stdout carries transcript JSON when selected.
Existing report directories and parents for regular transcript files must be
writable. Audita prevents report writes from overwriting transcript, glossary,
loaded application configuration, host YAML discovered under `profiles_dir`,
or host message files referenced by effective PromptKit `content_file` fields,
and rejects conflicting output destinations. Source paths come from the
runtime's captured configuration; unrelated files in the selected directory
remain eligible output destinations. The final transcript may intentionally
replace its own input after successful processing; other input/configuration/
profile sources remain protected.
Preparation never truncates an existing transcript. Device and pipe destinations
(such as `/dev/null` and named pipes) receive ordinary writes at publication time;
they are not opened during preflight or replaced with regular files. A pipe write
can wait for its reader, and stream write errors are fatal. With reporting enabled,
reports still use the output path's parent unless an explicit `--report-dir`
selects another directory; disable reporting when that placement is unsuitable
for a stream destination.
Before processing, Audita checks destination collisions against the transcript,
consumed application and PromptKit sources, and effective host message files. An
intentional in-place transcript replacement is allowed. A regular transcript
output is prepared in a sibling temporary file, leaving the final destination
untouched until successful publication. Pipes and devices are opened only for
delivery, and a pipe can wait for its reader. If a transcript destination changes
between a regular file and a pipe or device during processing, Audita can report a
delivery failure while preserving the new destination. Parent directories for
required outputs must already exist.
With reporting enabled, all three reports are initialized as `in_progress` before
input processing. Output is encoded fully before publication. Each regular output
file is replaced through a temporary file on its destination filesystem:
transcript first, then `errors.yml`,
`recoveries.yml`, and `report.md`. This is per-file replacement, not a
multi-file transaction. New files honor the process umask; replacing an existing
file preserves its permission bits. A late report error may leave the transcript
published; the process exits nonzero and attempts failed-status reports. On fatal processing
or cancellation, no completed transcript is published, but partial reports are
finalized where possible. An uncatchable termination can leave `in_progress`.
Enabled reports are initialized as `in_progress` before processing. The
transcript is published before final reports; the files do not form a single
transaction. A late report failure may therefore leave a published transcript.
Fatal errors and cancellation clean unpublished temporary resources and attempt
failed-status final reports. An uncatchable termination can leave `in_progress`
evidence. Read exit status and reports together rather than treating a file's
existence as proof of success.
## Diagnostics Directory
## Debug evidence
`--debug` or `diagnostics.debug` enables detailed evidence independently of
reporting. One directory named `run-<id>` is created under
`diagnostics.work_dir` (default `/tmp/audita`), which `--work-dir` can override,
and retained on success or failure. Selecting `--work-dir` alone does not enable
capture. Without debug, no directory or payload files are created. The parent
is created when needed only for debug. Failure to
write optional debug evidence prints a redacted stderr diagnostic but does not
change correction decisions or the exit status.
The shared `--debug` flag enables retained artifacts in a run directory beneath
`diagnostics.work_dir`; selecting a work directory alone does not enable capture.
Optional debug preparation or writes can produce redacted warnings without
changing correction decisions or the required output result. Debug material can
include transcript content, prompts, responses, and correction decisions. It is
not anonymized; protect it according to the input's sensitivity. Configured
credential values are redacted at the diagnostic boundary. Repair interactions
and revalidation artifacts include proposal and attempt coordinates; the
correction ledger connects them to the original edit.
Debug evidence includes invocation and effective settings, configuration
origins, selected profiles, redacted source and normalized snapshots,
normalization and initial chunk summaries, all captured proposed candidates
and validator decisions in `correction-ledger.json`, execution/utilization
records, and model interaction files. Individual artifact writes can fail; a
directory's existence does not prove every file was written. These files can
contain transcript text and prompt/response payloads. Configured backend
credentials are redacted, but the transcript itself is not anonymized. Protect
and retain the directory according to the sensitivity of the input.
## Recovery
## Failure Inspection and Recovery
Exit `0` means processing and every enabled required output completed. Exit
`1` means cancellation, fatal processing, or required-output failure. Exit `2`
identifies usage or configuration/destination selection errors. Recoverable
model-operation failures appear in `errors.yml` as one record per affected
work unit; they do not automatically make the run fail. `recoveries.yml`
records successful operations that needed retry, repair, or proposal filtering.
`report.md` shows every initial chunk and configured module instance, including
interrupted and unattempted work. Ordinary reports omit transcript excerpts and
full decision ledgers; enable debug when those are needed.
Read stderr and report status first. If a required output failed after a
transcript was published, do not treat the file alone as proof of a completed
run. Fix the input, service, or destination problem and rerun. Audita has no
resume/checkpoint API.
Exit `0` indicates completion of processing and required outputs; nonzero status
indicates an input, processing, cancellation, or required-output failure.
Recoverable model-operation failures and successful repairs or retries are
recorded separately in the ordinary YAML reports. The Markdown matrices identify
pipeline position, module, and optional operator ID. Fix the input, service, or
destination and rerun when necessary; Audita has no resume or checkpoint API.

View File

@@ -2,7 +2,12 @@
## Standard Repository Policies
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy Reading Guide in `docs/development.md`, adapted for this repository.
Material: All documents under `docs/policy/`, root `AGENTS.md`, and the Policy
Reading Guide in `docs/development.md`, adapted for this repository.
`docs/policy/documentation.md`, `docs/policy/oss.md`, and `docs/policy/release.md`
incorporate revision `593c9b52273c5c879785ae9e995d21026df467b4`.
The remaining listed material incorporates revision
`d84a38ebab26f048cafbe59a2246700c35d29843`.
Copyright (C) 2026 Eric Rakestraw
@@ -10,13 +15,17 @@ SPDX-License-Identifier: CC-BY-SA-4.0
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4).
## ConfigKit
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
Selected module: `go.maximumdirect.net/configkit` at `v1.2.0`.
Copyright (C) 2026 Eric Rakestraw
@@ -24,13 +33,16 @@ SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT).
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e/COPYRIGHT).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
[Source](https://forge.maximumdirect.net/base-layer/configkit/src/commit/c1c7e87f2cdf77a0cbf3327831ac05578e756b9e).
## ReleaseKit
Selected module: `gitea.maximumdirect.net/eric/ReleaseKit` at `v0.1.0`.
Selected module: `go.maximumdirect.net/releasekit` at `v0.9.0`.
Material: ReleaseKit runtime and adapted launcher and tag-delivery recipes in
`scripts/release.sh` and `.woodpecker/release.yml`.
Copyright (C) 2026 Eric Rakestraw
@@ -38,79 +50,121 @@ SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ReleaseKit/src/commit/5e60fe3113c69f95c42fbbe32a8ca4b51f5b83c1/COPYRIGHT).
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4/COPYRIGHT).
[Source](https://gitea.maximumdirect.net/eric/ReleaseKit/src/commit/5e60fe3113c69f95c42fbbe32a8ca4b51f5b83c1).
[Source](https://forge.maximumdirect.net/base-layer/releasekit/src/commit/2180aa868ec2caa95fc8279fd59d5f0ebac41bd4).
## CommandKit
Selected module: `go.maximumdirect.net/commandkit` at `v0.6.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/1613ff95cab60581de5d86f6e9e2a3cc091a44a0/COPYRIGHT).
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/1613ff95cab60581de5d86f6e9e2a3cc091a44a0).
## CommandKit ckconfig
Selected module: `go.maximumdirect.net/ckconfig` at `v0.7.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/64f6453b8453f914f272d427bb4ae6f59deacd2b/integrations/ckconfig/COPYRIGHT).
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/64f6453b8453f914f272d427bb4ae6f59deacd2b).
## CommandKit ckprompt
Selected module: `go.maximumdirect.net/ckprompt` at `v0.7.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0/integrations/ckprompt/COPYRIGHT).
[Source](https://forge.maximumdirect.net/base-layer/commandkit/src/commit/f369d6a78253b6e6fcb1e023fb2e9c3af1d20ff0).
## Go runtime and standard library
Selected toolchain: Go 1.25.5 for the distributed Linux binaries.
Selected toolchain: Go 1.27.3 for the distributed Linux binaries.
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/PATENTS).
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/PATENTS).
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
## Go vendored golang.org/x/crypto
Material: `golang.org/x/crypto` at `v0.39.0` in the Go 1.25.5 standard library.
Material: `golang.org/x/crypto` at `v0.52.1-0.20260526024921-9beb694f9766` in the Go 1.27.3 standard library.
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/crypto/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/crypto/PATENTS).
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/crypto/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/crypto/PATENTS).
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
## Go vendored golang.org/x/net
Material: `golang.org/x/net` at `v0.41.0` in the Go 1.25.5 standard library.
Material: `golang.org/x/net` at `v0.55.1-0.20260731170536-c1d18010be90` in the Go 1.27.3 standard library.
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/net/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/net/PATENTS).
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/net/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/net/PATENTS).
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
## Go vendored golang.org/x/sys
Material: `golang.org/x/sys` at `v0.33.0` in the Go 1.25.5 standard library.
Material: `golang.org/x/sys` at `v0.45.0` in the Go 1.27.3 standard library.
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/sys/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/sys/PATENTS).
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/sys/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/sys/PATENTS).
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
## Go vendored golang.org/x/text
Material: `golang.org/x/text` at `v0.26.0` in the Go 1.25.5 standard library.
Material: `golang.org/x/text` at `v0.37.0` in the Go 1.27.3 standard library.
Copyright 2009 The Go Authors.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/text/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/fefb02adf45c4bcc879bd406a8d61f2a292c26a9/src/vendor/golang.org/x/text/PATENTS).
[LICENSE](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/text/LICENSE) |
[PATENTS](https://github.com/golang/go/blob/2aae1000ea1d4b28923b96db28acf4e3b99d6aff/src/vendor/golang.org/x/text/PATENTS).
[Source](https://github.com/golang/go/commit/fefb02adf45c4bcc879bd406a8d61f2a292c26a9).
[Source](https://github.com/golang/go/commit/2aae1000ea1d4b28923b96db28acf4e3b99d6aff).
## go-yaml v4
@@ -156,7 +210,7 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt).
## PromptKit
Selected module: `gitea.maximumdirect.net/eric/promptkit` at `v0.12.0`.
Selected module: `go.maximumdirect.net/promptkit` at `v0.21.0`.
Copyright (C) 2026 Eric Rakestraw
@@ -164,37 +218,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL v3](licenses/LGPL-3.0.txt) |
[GPL v3](licenses/GPL-3.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3/COPYRIGHT).
[COPYRIGHT](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09/COPYRIGHT).
[Source](https://gitea.maximumdirect.net/eric/promptkit/src/commit/4b8169da286917fde7875425bb32b799b12c6af3).
## Promptkit OpenRouter Catalog
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` at `v3.20260930.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
## Promptkit Rakestrawhome Catalog
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` at `v3.20260930.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT-Catalogs.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/993443d88c451901f4c917fa00c204fd3b54d28d).
[Source](https://forge.maximumdirect.net/base-layer/promptkit/src/commit/171fb11f4e967a1da3af57843c6467954b14ba09).
## yaml.v3
@@ -216,3 +242,31 @@ License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
[NOTICE](https://github.com/go-yaml/yaml/blob/f6f7691b1fdeb513f56608cd2c32c51f8194bf51/NOTICE).
[Source](https://github.com/go-yaml/yaml/commit/f6f7691b1fdeb513f56608cd2c32c51f8194bf51).
## Cobra
Selected module: `github.com/spf13/cobra` at `v1.10.1`.
Copyright 2013-2023 The Cobra Authors
SPDX-License-Identifier: Apache-2.0
License and notices: [Apache 2.0](licenses/Apache-2.0.txt) |
[LICENSE](https://github.com/spf13/cobra/blob/7da941c3547e93b8c9f70bbd3befca79c6335388/LICENSE.txt).
[Source](https://github.com/spf13/cobra/tree/7da941c3547e93b8c9f70bbd3befca79c6335388).
## pflag
Selected module: `github.com/spf13/pflag` at `v1.0.9`.
Copyright (c) 2012 Alex Ogier. All rights reserved.
Copyright (c) 2012 The Go Authors. All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
License and notices: [BSD 3-Clause](licenses/BSD-3-Clause-Go.txt) |
[LICENSE](https://github.com/spf13/pflag/blob/10438578954bba2527fe5cae3684d4532b064bbe/LICENSE).
[Source](https://github.com/spf13/pflag/tree/10438578954bba2527fe5cae3684d4532b064bbe).

View File

@@ -1,18 +0,0 @@
MIT License
Copyright (c) 2026 Eric Rakestraw
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
associated documentation files (the "Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial
portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@@ -2,7 +2,7 @@
**Base Policy Version:** 1.3
**Last Updated**: 2026-09-28
**Last Updated**: 2026-11-11
**Upstream Source:** [Standard Repository Policies architecture](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/architecture.md)
## Purpose And Ownership
@@ -72,13 +72,19 @@ hidden global state.
## Correction and Concurrency Invariants
- Module focus guides proposal generation; it is not an exclusive acceptance
boundary. Validators permit otherwise acceptable minor editorial cleanup even
when the originating module did not request it. Existing applicability,
protection, and meaning checks still apply. See
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md) for the empirical
rationale and rejected strict-scope alternative.
- Treat LLM responses as nondeterministic inputs. Preserve deterministic proposal
indexing, validator ordering, transcript mutation, reports, and serialization.
- Validate corrections before final mutation, including local proposal checks,
decision cardinality, and apply-time safety. Provider-side schema enforcement
does not replace these checks.
- Keep LLM concurrency bounded and context-aware through PromptKit's engine-local
per-backend admission. Proposal and validation calls selecting the same backend
- Use PromptKit's context-aware, engine-local per-backend admission for
LLM concurrency. Proposal and validation calls selecting the same backend
share that capacity, including retries and repairs. Preserve deterministic
collection order. Recoverable model-operation failures stay local to their
proposal response or validation batch; fatal invariant/application failure
@@ -120,13 +126,19 @@ responsibilities and should remain separate. Implementation details belong in
## Dependencies
Keep Audita dependency-light and prefer the Go standard library for CLI parsing,
filesystems, synchronization, and tests. ConfigKit owns configuration loading,
presence, provenance, and secret resolution; PromptKit owns prompt/profile
loading, OpenAI-compatible transport, retries, repairs, and backend admission.
Audita owns domain meaning, role selection, and deterministic processing.
Keep Audita dependency-light and prefer the Go standard library where no adopted
library owns the boundary. ConfigKit owns configuration composition, presence,
recorded resolution, and provenance. PromptKit owns prompt, profile, backend,
credential, transport, retry, repair, and admission behavior. CommandKit and its
integrations own command routing and their standard inspection and request
surfaces. Audita owns domain meaning, role selection, transcript processing,
and presentation of its processing results.
Do not introduce broad CLI, dependency-injection, workflow, logging, or plugin
Pass resolved typed configuration to preflight and processing. Preflight checks
execution readiness before output preparation or model work; the output session
owns resources, cleanup, and publication. The glossary and readiness decision is
recorded in [ADR 0002](../adr/0002-glossary-configuration-and-preflight.md).
Do not introduce broad dependency-injection, workflow, logging, or plugin
frameworks without a concrete implemented need and focused tests.
## Testing and Documentation

View File

@@ -1,9 +1,9 @@
# Documentation Policy
**Base Policy Version:** 1.11
**Base Policy Version:** 1.12
**Last Updated**: 2026-09-28
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/documentation.md)
**Last Updated**: 2026-11-11
**Upstream Source:** [Standard Repository Policies documentation](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/documentation.md)
## Purpose
@@ -85,8 +85,8 @@ secret values.
| Inventory of implemented internal packages | `docs/internal/overview.md` | Implemented packages and components, responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
| Behavior and architecture of internal packages | Other files under `docs/internal/` | Implementation flow, collaborators, state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
| Contributor change recipes | Relevant files under `docs/internal/`, including `docs/internal/cli-config.md`, `docs/internal/modules.md`, and `docs/internal/validators.md` | Concrete editing and validation steps for each component. | Contributor orientation, global policy, and definitions of external contracts. |
| Architectural decision records | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
| Architectural decision records | `docs/adr/` | Records of significant architectural decisions and supporting rationale. | Ordinary design choices, current behavior reference, implementation status, task sequencing. |
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, or future work; implementation status, sequencing, and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
| Release-note history | `docs/releases/` | One checked-in summary per source release under the procedure; the note at its immutable tag is the release record. | Current commands, behavior, contracts, and compatibility definitions. |
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other copyable or runnable files. | Field-by-field reference, command reference, prose explanation. |
@@ -154,8 +154,15 @@ Use sequentially numbered ADR filenames such as
Treat the decision content of an accepted ADR as immutable. When a decision
changes, create a new ADR and update the earlier ADR's status to superseded.
Rejected architectural alternatives belong in the ADR; rejected product ideas
belong in the roadmap.
ADRs should be reserved for major architectural decisions, such as ownership
boundaries, large-scale architectural features, or invariants that rise to
the level of architectural significance. Routine choices, implementation
decisions, and rejected suggestions do not warrant an ADR unless they
otherwise represent a significant architectural decision.
Roadmaps should preserve plausible future work, not rejected product
ideas.
## Maintenance

View File

@@ -1,9 +1,9 @@
# Open Source Licensing And Attribution Policy
**Base Policy Version:** 1.8
**Base Policy Version:** 1.9
**Last Updated**: 2026-09-28
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/oss.md)
**Last Updated**: 2026-11-11
**Upstream Source:** [Standard Repository Policies oss](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/oss.md)
## Purpose And Scope
@@ -64,9 +64,12 @@ license unless the author deliberately changes it. For mixed-purpose projects,
follow the author's selected scope rather than inferring different licenses for
individual packages.
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/application/COPYRIGHT)
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/library/COPYRIGHT)
Use the upstream [application COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/application/COPYRIGHT)
or [library COPYRIGHT example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/library/COPYRIGHT)
to create the project’s root `COPYRIGHT` file.
Use the COPYRIGHT examples exactly as provided, except for placeholder
substitutions and changes expressly required or authorized by this policy or
the author.
Use verified project names, contribution years, and copyright holders.
For another selected license, adapt the grant, SPDX expression, disclaimer, and
text references together to reflect the author's selected grant.
@@ -113,7 +116,7 @@ Each entry presents these five fields in order, with blank lines between them:
filenames, such as `COPYRIGHT`, `LICENSE`, `NOTICE`, or `PATENTS`.
5. **Source repository link**, rendered exactly as `[Source](COMMIT_URL).`.
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/examples/licensing/attribution.md)
Use the upstream [attribution example](https://gitea.maximumdirect.net/eric/policy/src/commit/a469273adda336625f96b9d769ea27b33999f146/examples/licensing/attribution.md)
to create the project’s `docs/oss/attribution.md`.
The example illustrates omitted fields and multiple copyrights and licenses.
Replace the uppercase placeholders and illustrative URLs with actual component

View File

@@ -1,9 +1,9 @@
# Release Policy
**Base Policy Version:** 1.4
**Base Policy Version:** 1.7
**Last Updated**: 2026-10-07
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
**Last Updated**: 2026-11-11
**Upstream Source:** [Standard Repository Policies release](https://gitea.maximumdirect.net/eric/policy/src/commit/593c9b52273c5c879785ae9e995d21026df467b4/docs/policy/release.md)
## Purpose
@@ -11,6 +11,9 @@ This policy defines a proportionate workflow for validating and publishing an
intended source revision under a fresh version. Releases should require little
routine intervention, avoid repeated work, and preserve published contents.
Adopt stronger verification or reproducibility requirements through an explicit
project-specific decision justified by the intended audience and support needs.
The repository-specific commands belong in `docs/release.md`. Historical release
summaries belong in `docs/releases/`. The procedure must agree with the adopted
policy and link to it rather than introduce additional release requirements.
@@ -75,9 +78,18 @@ requires a deliberate update to the policy and procedure where needed.
For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and
- a minor release may add backward-compatible behavior or raise the Go consumer
minimum as described below; and
- a patch release may make backward-compatible fixes.
In this context, "backward-compatible" means only that pre-existing, valid
configurations will continue to work following the release. Behavior need not
remain strictly identical.
Audita's minor and patch releases also preserve the documented
[subprocess integration](../integrations/subprocess.md) and
[output-file](../integrations/output-files.md) contracts.
Before `v1.0.0`, a minor release may change a documented public, operational,
configuration, data, integration, or compatibility contract when the release
note explains the impact and required consumer action. A patch release must
@@ -87,6 +99,36 @@ Additional compatibility promises and deliberate exceptions belong in the
adopted policy or canonical contract documentation. Release notes explain
changes; they do not redefine current compatibility contracts.
## Consumer Minimum And Release Toolchain
Distinguish the minimum toolchain supported for consumers from the toolchain
maintainers use to build and validate a release. Record the consumer minimum in
the language's standard metadata and explain its support meaning in canonical
documentation. Define the selected release toolchain and its setup commands in
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
choices.
For Go repositories, normally declare the initial stable release of the selected
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
when a dependency or an identified correctness fix requires it, and document
the reason.
For release-policy purposes, successful build and release validation on the
selected Go patch release are sufficient to presume support across that Go
series, subject to the declared consumer minimum and concrete evidence of
incompatibility. Investigate specific compatibility concerns as they arise.
Use the selected release toolchain for candidate validation and release builds,
including CI jobs performing those tasks. A newer validation patch version alone
leaves the consumer minimum unchanged.
A minor release may raise the Go consumer minimum, including from one Go series
to the next, both before and after `v1.0.0`. This is an explicit exception to the
backward-compatibility requirement for minor releases. Patch releases preserve
the consumer minimum. Explain an increase's impact and required consumer action
in the release note. Other incompatible changes remain subject to the versioning
rules above.
## Release Documentation And Preparation
Every new release must include `docs/releases/<version>.md` in the tagged
@@ -145,9 +187,19 @@ caches are not required. If the candidate changes, validate the new candidate.
Cheap publication-state checks may be repeated immediately before pushing
because local or remote state may have changed during validation.
Automate mechanical guards. Do not require manual approvals, sign-offs, or
routine confirmations as release gates. Version selection and
a concise release summary remain preparation inputs, not separate approval rounds.
Automate mechanical guards. Human review gates are strongly disfavored: add one
only through an explicit project-specific policy decision explaining the
concrete risk and why ordinary change review or automation is insufficient.
Do not add routine confirmations for checks the command can perform itself.
## Audita's Selected Candidate Checks
Audita uses ReleaseKit v0.9.0 with race-enabled offline tests, full Go vet,
formatting, Markdown links, module distribution checks, and the required target
builds. Race checks address shared processing and client state; vet and format
checks catch common defects and mechanical drift; link and module checks protect
the distributed source tree. The [runbook](../release.md) specifies the selected
toolchain, native commands, and asynchronous recovery.
## Publication Guards
@@ -201,7 +253,8 @@ release without replacing its contents or reusing its version.
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
identify:
1. prerequisites, primary branch, and authoritative remote;
1. prerequisites, selected release toolchain, primary branch, and authoritative
remote;
2. version and release-note preparation;
3. one publication command with its selected local checks and automated guards;
4. declared deliverables, required build targets, and automation responsibilities;

View File

@@ -2,16 +2,37 @@
The [release policy](policy/release.md) owns release requirements. This runbook
uses the native commands in [scripts/release.sh](../scripts/release.sh), which
builds and invokes ReleaseKit. A confirmed source tag completes the release;
Woodpecker reports binary delivery separately.
invokes the module-selected ReleaseKit tool. A confirmed source tag completes
the release; Woodpecker reports binary delivery separately.
## Prerequisites
Use Bash, Git, and Go 1.25.5 or newer. The authoritative remote is `origin`,
which must point to `https://gitea.maximumdirect.net/eric/audita.git`; the
Use Bash, Git, a C compiler, and cgo for race detection. The authoritative
remote is `origin`, which must point to `https://gitea.maximumdirect.net/eric/audita.git`; the
primary branch is `main`. Source publication needs no delivery credential.
Woodpecker uses the existing `GITEA_RELEASE_TOKEN` secret for delivery.
The source-build minimum is Go 1.27.0, declared in [go.mod](../go.mod).
This minimum is required by the selected application dependencies.
Distributed executables do not require a Go installation. The selected release
toolchain is Go 1.27.3, matching the current Woodpecker image and distributed
runtime notices. Use that local installation for tool bootstrap, candidate
validation, and release builds. Replace the installation path below with its
actual directory:
```sh
export PATH="/path/to/go1.27.3/bin:$PATH"
export GOTOOLCHAIN=local
go version
```
Record the reported toolchain with the validation results. The
[release policy](policy/release.md#consumer-minimum-and-release-toolchain)
governs the distinction between the consumer minimum and release toolchain,
support across a Go series, and versioning of minimum increases. When selecting
a different release toolchain, update the CI image and applicable runtime
attribution together; a newer build patch alone need not raise the minimum.
Prepare dependencies before offline candidate validation:
```sh
@@ -19,9 +40,13 @@ GOWORK=off GOTOOLCHAIN=local go mod download
```
If dependency preparation changes `go.mod` or `go.sum`, include those changes in
the committed candidate. The release wrapper separately bootstraps the pinned
ReleaseKit command into ignored `bin/releasekit`; that build is tool preparation,
not candidate validation.
the committed candidate. ReleaseKit's CLI is registered as a Go tool dependency;
`go mod tidy` retains its dependencies and checksums. The release wrapper changes
to the repository root, selects the host target, and invokes the full
`go.maximumdirect.net/releasekit/cmd/releasekit` tool path with workspace
discovery and automatic toolchain downloads disabled. Go builds or reuses the selected
tool from its normal build cache. Tool preparation precedes offline candidate
validation.
## Prepare and publish the candidate
@@ -54,9 +79,11 @@ Publish from the repository root, substituting the selected version:
bash scripts/release.sh publish VERSION
```
The command runs `go test -mod=readonly -count=1 ./...` once with module and
toolchain network access disabled, builds the profile's Linux amd64 and arm64
targets, and checks the native version output. It creates an annotated tag whose
The command runs the offline Go test suite with race detection, full Go vet,
formatting, Markdown link, and module distribution checks. Race detection covers
shared processing and client state; vet and formatting catch common Go defects and
mechanical drift; the link and module checks protect the distributed source tree. It builds the
profile's Linux amd64 and arm64 targets and checks the native version output. It creates an annotated tag whose
message comes from the committed note, then pushes the exact tag object. A
successful publication completes the source release without waiting for CI or
checking artifact availability.
@@ -71,33 +98,46 @@ profile, or version input error; and `3` means the push outcome remains unknown.
## Binary delivery
Woodpecker runs the pinned ReleaseKit `ci-deliver` command for tags under
`refs/tags/v*`. It verifies the triggering commit against the tag, prepares the
tagged source, and uses `releasekit.json` to build and deliver:
Woodpecker runs ReleaseKit's `ci-deliver` command for tags under
`refs/tags/v*`. The tagged `go.mod` selects the ReleaseKit version, and
`releasekit.json` supplies Audita's build and delivery settings:
- `audita-<version>-linux-amd64`
- `audita-<version>-linux-arm64`
- `SHA256SUMS`
The workflow invokes:
Before running Go, the workflow fetches and verifies the event tag, checks it
out detached, and confirms the source tree is clean. It peels the event object
ID to its commit before comparing it with the tag's commit, accepting both
annotated tag objects and commit IDs from Woodpecker. It copies the tagged
`go.mod` and `go.sum` into temporary module metadata outside the checkout. Go
confirms that this configuration selects ReleaseKit, then invokes its CLI
using an alternate `-modfile` and the host target. Temporary metadata remains
outside the tagged source. The workflow runs with the selected Go 1.27.3 image,
`GOWORK=off`, and `GOTOOLCHAIN=local`.
Delivery uses prepared-checkout mode to verify the existing tag, HEAD, and clean
worktree without repeating the bootstrap's fetch and checkout.
The resulting executable receives:
```sh
go run gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit@${RELEASEKIT_VERSION} ci-deliver \
go tool -modfile="$${bootstrap_dir}/releasekit.mod" "$${releasekit_module}/cmd/releasekit" ci-deliver \
--repo "$${CI_WORKSPACE}" \
--commit "$${CI_COMMIT_SHA}" \
--prepared \
--commit "$${expected_commit}" \
"$${CI_COMMIT_TAG}"
```
Woodpecker sets `RELEASEKIT_VERSION` to `v0.1.0` for this workflow.
The complete checked-in release note, including its licensing footer, supplies
the hosted release body. The workflow maps the existing `GITEA_RELEASE_TOKEN`
secret to `RELEASEKIT_GITEA_TOKEN`. Delivery runs independently after source
publication; its status and logs report delivery progress or failure.
The workflow maps the existing `GITEA_RELEASE_TOKEN` secret to
`RELEASEKIT_GITEA_TOKEN`. The complete checked-in release note, including its
licensing footer, supplies the hosted release body. Delivery runs independently
after source publication; its status and logs report delivery progress or
failure.
ReleaseKit handles content-aware retries and retained-artifact recovery. See
the pinned [publication recovery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/publish.md)
and [delivery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/delivery.md)
the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md)
and [delivery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md)
for those mechanics. A delivery failure does not undo a confirmed source
release. Retry the same immutable tag after resolving the reported state; do not
replace published files or a checksum manifest.
@@ -110,11 +150,11 @@ captured authoritative endpoint and publication state. Resolve the result
against that endpoint before retrying. An exact remote tag-object match, with
its source available on remote `main`, can confirm publication. If the remote
state cannot be read, the result remains unknown; do not remove or replace the
tag while uncertain. Follow the pinned [publication recovery guide](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/publish.md)
tag while uncertain. Follow the pinned [publication recovery guide](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/publish.md)
for object-level recovery details.
A source-release success and artifact-delivery status are separate. For a
partial or uncertain delivery, inspect the job output and follow the pinned
[delivery recovery guidance](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/delivery.md).
[delivery recovery guidance](https://forge.maximumdirect.net/base-layer/releasekit/src/tag/v0.9.0/docs/consumers/delivery.md).
A source or published-content correction requires a new appropriate version;
leave the original tag and its contents intact.

View File

@@ -1,5 +1,7 @@
# Release Notes
- [Audita v3.1.1](v3.1.1.md)
- [Audita v3.1.0](v3.1.0.md)
- [Audita v3.0.1](v3.0.1.md)
- [Audita v3.0.0](v3.0.0.md)
- [Audita v2.0.1](v2.0.1.md)

View File

@@ -38,8 +38,8 @@ profile files require migration from the v2 release format.
3. Select a configuration root whose final definitions are valid. PromptKit now
validates unused definitions too, including backend references, and conflicting
values within a directory source group are errors. The maintained examples
use separate [local](../../examples/profiles/local/) and
[service](../../examples/profiles/service/) roots. Runtime source edits take
use separate [local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/local/) and
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/profiles/service/) roots. Runtime source edits take
effect on the next invocation.
4. Rename any application backend literally named `extends` and update its
profile references. ConfigKit now reserves that key for inheritance.
@@ -47,11 +47,11 @@ profile files require migration from the v2 release format.
`audita config validate --config /path/to/config.yml`, including any selected
testing overlay. Application configuration continues to use `version: 2`.
The [configuration reference](../config.md#promptkit-profiles-and-operator-prompts)
contains the migrated format and presence rules. Start from the maintained
[local](../../examples/minimal-config.yml),
[service](../../examples/production-config.yml), or
[application inheritance](../../examples/configkit-inheritance.yml) examples.
The [v3.0.0 configuration reference](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/docs/config.md#promptkit-profiles-and-operator-prompts)
contains the migrated format and presence rules. Start from the
[local](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/minimal-config.yml),
[service](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/production-config.yml), or
[application inheritance](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.0.0/examples/configkit-inheritance.yml) examples.
The [CLI reference](../cli.md) describes report selection, and the
[release procedure](../release.md) describes source publication and asynchronous
Linux binary delivery.

42
docs/releases/v3.1.0.md Normal file
View File

@@ -0,0 +1,42 @@
# Audita v3.1.0
This minor release updates Audita's configuration, LLM, and release libraries
and raises the minimum Go version for source builds to 1.26.0. Existing Audita
configuration files, PromptKit profiles, and command behavior require no
migration.
## Changes
- ConfigKit is updated from v0.5.0 to v0.7.1, with improvements to configuration
loading, resolution, provenance, and inspection efficiency.
- PromptKit is updated from v0.12.0 to v0.14.0, reducing avoidable allocations in
preparation, rendering, and execution. Audita retains its existing schemas,
validation, retry and repair budgets, and resilient processing behavior.
The libraries' new discovery and configuration-comparison APIs remain unused
by Audita.
- ReleaseKit is updated from v0.1.0 to v0.4.1. The local release launcher now
invokes the module-selected Go tool, and CI selects the tool from the tagged
module metadata. Prepared-checkout delivery reuses the tag checkout already
verified during CI bootstrap. Source publication still completes at a
successful tag push; Linux binary delivery remains asynchronous.
- Release builds and CI use Go 1.26.8, with corresponding runtime attribution
updated. The source-build minimum is tracked separately from that build
toolchain patch version.
## Upgrading from v3.0.1
Source builders must upgrade from the previous Go 1.25.5 minimum to **Go 1.26.0
or newer**, including any build environments still using Go 1.25. Maintainers
should follow the updated [release procedure](../release.md) for toolchain setup,
publication, and delivery recovery.
Go module consumers must use the v3 module path. For example:
`go install gitea.maximumdirect.net/eric/audita/v3/cmd/audita@v3.1.0`. This
aligns the module path with the v3 release line. Downloaded binaries and
commands run from a source checkout are unaffected.
Users of distributed Linux binaries need no Go installation. No configuration,
profile, transcript-format, or command-behavior migration is required for this
release.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.0/docs/oss/attribution.md)

20
docs/releases/v3.1.1.md Normal file
View File

@@ -0,0 +1,20 @@
# Audita v3.1.1
This patch fixes the Woodpecker bootstrap failure that prevented Linux binary
delivery for v3.1.0. Audita's transcript processing and configuration are unchanged.
## Changes
- Resolve Woodpecker's tag-event object ID to its source commit before checking
the tagged checkout and invoking ReleaseKit. Annotated tag events previously
failed a comparison between the tag object and commit, stopping the workflow
before binary builds began.
- Add offline regression coverage for annotated and lightweight tag events,
including rejection of an unrelated source commit.
No configuration, profile, transcript-format, or command-behavior migration is
required from v3.1.0. The Go source-build minimum remains 1.26.0. Linux amd64 and
arm64 binary delivery follows the existing asynchronous
[release procedure](../release.md#binary-delivery).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/audita/src/tag/v3.1.1/docs/oss/attribution.md)

View File

@@ -1,133 +0,0 @@
# ConfigKit v0.5.0 Adoption Roadmap
**Last updated:** 2026-10-03
**Status:** Complete. The implementation and tests are committed through
`8175ed8`. Final documentation and example changes were included in the full
build, test, example-validation, and local-link checks before the documentation
completion commit. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/b31955fc97adec707b4e5f9a1b0c43c4cbc66750/docs/roadmap/implementation.md)
records the ordered work and verification.
## Purpose and Baseline
Adopt ConfigKit v0.5.0 as an independently reviewable step before PromptKit
v0.12.0. Use the library's configuration and inheritance conventions while
preserving Audita's domain ownership and existing operator workflows.
At the reviewed Audita revision `6ae1d7cd5f548b26de5787372d3f2249eaa4757d`,
[go.mod](../../go.mod) selects ConfigKit v0.4.2, PromptKit v0.11.2, and Go
1.25.5. ConfigKit already owns loading, presence, secret resolution, and
configuration inspection. Audita's `internal/core/config` supplies typed
settings, defaults, derived values, and domain validation; `internal/cli`
selects files, overlays, and explicit overrides. The
[configuration contract](../config.md) and [CLI contract](../cli.md) own the
implemented behavior.
The reviewed upstream release is
[ConfigKit v0.5.0](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/releases/v0.5.0.md),
commit `dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It adds layered underlays,
caller-owned filesystem sources, mapping sources, source resource contexts, and
built-in sibling inheritance. The Go minimum and YAML dependency do not change.
Ordinary `Load` remains available; adopting `LoadLayered` is optional.
This roadmap supersedes the completed original adoption plan at this path.
The [historical roadmap](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md)
records that earlier migration. Its coordinated breaking-refactor instructions
and release target do not govern this incremental upgrade.
## Target End State
- Audita uses the released ConfigKit v0.5.0 module, with no local replacement.
PromptKit stays at v0.11.2 throughout this adoption unit.
- Configuration retains the current file-selection, overlay, explicit-override,
resolution, and validation flow, subject to the reserved-key compatibility
impact described below. Retain the application schema version, CLI interface,
and main-file-relative resource rules defined in the current contracts.
- Application configuration can use ConfigKit's native sibling `extends`
relationships. Audita does not implement a second inheritance resolver,
special backend-only syntax, or an opt-in switch around the library feature.
- Inherited values participate in presence-aware defaults, domain validation,
secret resolution, and safe inspection just like other effective input.
In particular, inherited backend settings must not be overwritten merely
because they were not written directly on the child.
- Effective settings and explanations remain useful and secret-safe when
values are inherited. Preserve existing inspection fields and add compact,
value-free inheritance evidence from ConfigKit's metadata rather than
reconstructing provenance from YAML.
- Canonical documentation and a maintained example explain the adopted
inheritance behavior. Attribution identifies the exact new upstream source.
The upstream [configuration contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.5.0/docs/config.md#sibling-inheritance)
owns generic inheritance mechanics. The integration must account for three
consequences: `extends` is reserved automatically throughout input mappings;
relationships resolve after source precedence and before typed decoding; and
inherited zero, false, empty, or null values count as explicit input. An inherited
value still has to satisfy Audita's field-specific rules. Parent definitions
also undergo ordinary application validation; inheritance does not introduce
abstract, validation-exempt backend entries.
## Scope and Adoption Boundary
Keep the existing `Load` integration unless implementation reveals a concrete
need to change it. Source-backed defaults, additional operator underlay tiers,
imports, and source-relative reinterpretation of application paths are outside
this feature. Their availability upstream does not require expanding Audita's
configuration surface. Imports and secret-directory bootstrap retain their
current disabled state.
ConfigKit inheritance applies to Audita configuration in this step. PromptKit
profile formats, profile inheritance, engine construction, embedded prompts,
and backend catalogs remain governed by the existing PromptKit integration.
There is no application-owned translation of profile documents in this work.
This is an independent ConfigKit adoption ahead of the planned PromptKit
migration. A separate public release between the two units is optional.
## Acceptance Criteria
- The released ConfigKit v0.5.0 dependency and exact source attribution are
adopted; PromptKit remains at v0.11.2 with its existing catalogs.
- Native inheritance preserves effective endpoints, capacity, explicit zero
queue capacity, child overrides, and surviving credential references through
Audita's loading, resolution, and validation boundary.
- Selected overlays and explicit overrides retain their precedence; omitted
validation-role selection still derives from the final proposal selection.
- Null can remove a child's credential while surviving references on parents
and other unselected backends still follow the existing credential contract.
- Selected roles use inherited backend settings and share its configured
capacity under the existing PromptKit runtime contract.
- Effective settings remain redacted, and both CLI explanations and debug
configuration origins expose compact, value-free inheritance evidence while
retaining existing fields and application-derived explanations.
- A maintained complete example and canonical documentation describe native
inheritance, its field-validation implications, and the reserved-key
compatibility impact. Existing workflows and applicable offline checks pass.
## Completion and Release Treatment
Adoption is complete when the released dependency is pinned, existing workflows
pass, representative inheritance works through loading and runtime wiring,
inspection includes value-free inheritance evidence and remains secret-safe,
and the example, canonical docs, and attribution match the implementation. No
PromptKit migration or provider behavior change is required to close this unit.
Release classification remains provisional. A minor release would be appropriate
for a backward-compatible addition, but the newly reserved `extends` key can
invalidate currently valid application configuration. For example, Audita
currently permits a backend literally named `extends`; ConfigKit v0.5.0 treats
that mapping key as an inheritance directive rather than a backend ID. Do not
promise blanket compatibility or settle on a minor release until the final
compatibility review resolves this impact under the
[release policy](../policy/release.md). Describe any required migration in the
canonical documentation and release notes when the upgrade ships. Do not select
or publish a release as part of roadmap preparation. A later PromptKit
migration's profile-format compatibility decision belongs to that separate
feature.
## Selected Overlays for Testing
This heading preserves links from historical adoption documents. The original
[overlay plan](https://gitea.maximumdirect.net/eric/audita/src/commit/6ae1d7cd5f548b26de5787372d3f2249eaa4757d/docs/roadmap/configkit.md#selected-overlays-for-testing)
is historical; current overlay behavior belongs to [configuration](../config.md).
This upgrade preserves that workflow and adds native ConfigKit inheritance
without reviving the earlier combined migration plan.

View File

@@ -1,19 +1,20 @@
# Deferred Feature Work
**Last updated:** 2026-10-03
**Last updated:** 2026-11-11
**Status:** Deferred or exploratory work; not implemented or scheduled. These
entries preserve work beyond the completed resilience and span-deletion
features. Each entry states its intent; inclusion does not commit us to
entries preserve work beyond completed feature and library upgrades.
Each entry states its intent; inclusion does not commit us to
implementing every capability.
## Domain-Aware Response Recovery
The implemented [partial-success decision reconciliation](../internal/validators.md#llm-validator-batching)
preserves usable decisions without another model call. Guided recovery remains
deferred pending evidence from failures that remain. General per-entry
salvage from structurally invalid responses is also a separate design question;
the accepted reconciliation feature retains whole-response structural validation.
preserves usable decisions without another model call. Revision of rejected
correction proposals is implemented with substantive
validator feedback and explicit proposal withdrawal; see the
[pipeline guide](../internal/pipeline.md).
It is distinct from recovering an unavailable validator decision.
Consider bounded feedback and regeneration for model responses that satisfy the
structural schema but leave requested decisions unavailable after reconciliation.
@@ -30,11 +31,15 @@ secrets, and how regeneration fits bounded budgets and operation deadlines. Reus
shared library capabilities where appropriate; do not introduce an uncoordinated
outer retry loop that multiplies existing retry/repair budgets.
Ordinary editorial rejection is a valid decision, not a reason to regenerate until
an edit is approved. Recovery must preserve required validation, candidate identity,
partial-success safety, and truthful error/recovery reporting. Validate with
controlled responses that distinguish recoverable protocol defects from valid
negative decisions. Exact APIs, budgets, and sequencing remain undecided.
Editorial rejection remains a valid decision about the evaluated proposal. The
implemented correction repair loop handles subsequent revision and full
revalidation; this deferred entry concerns unavailable decisions. Decision recovery
must preserve required validation, candidate identity, partial-success safety,
and truthful error/recovery reporting. Validate with controlled responses that
distinguish protocol defects from valid negative decisions. Exact APIs, budgets,
and sequencing remain undecided. General per-entry salvage from structurally
invalid responses also remains a separate design question; current reconciliation
retains whole-response structural validation.
## Independent Whole-Pipeline Traversal by Chunk

View File

@@ -1,327 +0,0 @@
# Implementation Plan: ReleaseKit Adoption
**Last updated:** 2026-10-07
**Status:** Complete; Stages 1–5 complete.
**Audience:** Luna coding agent.
## Scope and Execution
Implement the target in the [ReleaseKit roadmap](releasekit.md), following
[ADR 0001](../adr/0001-releasekit.md). The roadmap owns intent and policy
choices; this document owns execution. The planning baseline is Audita commit
`418da456d75a68d3bb5a43ab823c5b2b434b90cc`: v3.0.0 is released, the ADR is
accepted, and release tooling still uses the custom scripts and delivery plugin.
This replaces the completed PromptKit plan. Its
[completion record](https://gitea.maximumdirect.net/eric/audita/src/commit/418da456d75a68d3bb5a43ab823c5b2b434b90cc/docs/roadmap/implementation.md)
remains available for historical adoption references.
Execute Stage 1 through Stage 5 in order. Each is a bounded Luna implementation
unit. Commit completed units with plain English messages under the
[agent policy](../policy/agents.md), recording the relevant checks. Intermediate
commits need not be release-ready; complete the migration directly without
compatibility wrappers for the old release arguments or temporary release modes.
Start with [development guidance](../development.md), use graph-first discovery,
and follow the applicable [architecture](../policy/architecture.md),
[testing](../policy/testing.md), [documentation](../policy/documentation.md),
[release](../policy/release.md), and [OSS](../policy/oss.md) policies. Preserve
transcript processing and its current configuration, reporting, and LLM behavior.
Use ReleaseKit v0.1.0, commit
`5e60fe3113c69f95c42fbbe32a8ca4b51f5b83c1`. The sibling checkout is reference
material. Its pinned [configuration](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/config.md),
[CLI](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/cli.md),
[build](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/build.md),
and [delivery](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/docs/consumers/delivery.md)
contracts own generic mechanics. Keep Go 1.25.5 and unrelated dependency
selections. Use released modules without a production local replacement.
### Selected Implementation Decisions
- Audita imports `ReleaseKit/buildinfo` for runtime version information. The
upstream executable owns release orchestration; no application release engine
or private linker variable is needed.
- `releasekit.json` owns product facts. Local tooling builds the executable from
the module graph; CI pins the same v0.1.0 release explicitly.
- Keep `scripts/release.sh` as a thin entry point with native ReleaseKit
arguments, such as `publish VERSION` and `check VERSION`. Compile its tool into
ignored `bin/releasekit` and invoke it with `exec` to preserve native statuses.
- Main-first preparation, annotated tags, the offline standard suite, Linux
builds, and the native version smoke check are accepted. On a nonselected
host, the shared builder may compile a temporary native executable and require
its check to succeed. Keep only Linux amd64 and arm64 delivery targets.
- CI uses managed `ci-deliver` and the existing `GITEA_RELEASE_TOKEN` secret,
exposed to the command as `RELEASEKIT_GITEA_TOKEN`. Retained prebuilt artifacts
remain an upstream recovery option rather than a second ordinary CI sequence.
- Additional profile checks are omitted. Do not repeat the standard suite,
maintain a local Git/publication guard layer, or copy upstream delivery logic.
Focus tests on Audita wiring and reuse existing protection for common mechanics.
This implementation sprint does not choose an Audita release version, write a
new versioned release note, tag, push to the authoritative repository, contact
Gitea with delivery credentials, or wait for CI. Deployment observations belong
to the first adopting release and its separate delivery status. No extra release
gates or platform guarantees are introduced by these adoption checks.
## Stage 1 — Add the Dependency and Version Command [x]
**Primary files:** `go.mod`, `go.sum`, `internal/cli/run.go`, and focused tests in
`internal/cli`.
1. Add `gitea.maximumdirect.net/eric/ReleaseKit@v0.1.0` with `go get`. Import its
public `buildinfo` package in the CLI and run `go mod tidy` after the import
is present. Review the dependency diff for unrelated upgrades or replacements.
2. Add `version` dispatch to the existing CLI. With no arguments, print
`audita ` followed by `buildinfo.Version()` and a newline to stdout, returning
zero. Run this route independently of application configuration and runtime
assembly. Use the existing output streams and standard-library parsing style.
3. Add the command to root help. Support `help version` and the usual
`version -h`/`version --help` help behavior with stdout and status zero.
Unexpected positional arguments or unknown flags use the existing usage-error
status two. No new version flags are needed at the root level.
4. Add compact CLI coverage for successful version output, help, and malformed
invocation. Derive the expected version from the public accessor rather than
hardcoding `dev`. Exercise the version route from a temporary working directory
without application setup and verify stderr stays empty on success. Preserve
existing commands and help behavior.
**Verification and completion:** Run
`go test ./internal/cli ./cmd/audita`, build the application to ignored
`bin/audita`, and run its `version` command. Run `git diff --check`, review the
module diff, and commit. Real release-version injection is verified in Stage 5;
no test should hardcode an upstream private linker symbol.
**Completed 2026-10-07:** Added ReleaseKit v0.1.0 and the configuration-free
`audita version` command with root/help routing and usage-error handling.
Checks passed: `go test ./internal/cli ./cmd/audita`,
`go build -o ./bin/audita ./cmd/audita`, `./bin/audita version`, and
`git diff --check`.
## Stage 2 — Introduce the Product Profile and Thin Local Wrapper [x]
**Primary files:** new `releasekit.json`, `scripts/release.sh`, and
`scripts/release_test.go`.
1. Write a JSON profile with these exact product settings:
| Field | Value |
| --- | --- |
| `product` | `Audita` |
| `remote` | `origin` |
| `build.main_package` | `./cmd/audita` |
| `build.binary_name` | `audita` |
| `build.targets` | `linux/amd64`, `linux/arm64` |
| `delivery.base_url` | `https://gitea.maximumdirect.net` |
| `delivery.owner` | `eric` |
| `delivery.repository` | `audita` |
| `delivery.token_env` | `RELEASEKIT_GITEA_TOKEN` |
Omit `checks`. Store no token value and use the upstream profile format
directly; Audita does not need another profile parser in production.
2. Replace the wrapper body with straightforward Bash using `set -euo pipefail`:
select the repository directory relative to the script, create `bin/`, build
the selected CLI, and execute it with the original argument vector. The
build command is:
```sh
GOWORK=off GOTOOLCHAIN=local go build -mod=readonly \
-o ./bin/releasekit gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit
```
Finish with `exec ./bin/releasekit "$@"`. Preserve argument boundaries and
build failures. Existing `bin/` ignore rules suffice. Let ReleaseKit parse
its commands and report errors; remove all local note, version, Git, test,
build-target, and publication mechanics from the wrapper.
3. Replace `TestReleaseWorkflow`'s old fourteen-scenario matrix. Use a temporary
layout with the real wrapper and a stub at the Go/tool boundary to verify
repository selection, forwarded native arguments (including an argument with
spaces), and returned tool status. Include uncertain publication status three
and a failed bootstrap that never executes the tool. The stub writes a small
executable at the requested output path; assert effects and arguments without
tying tests to harmless build-flag ordering or a particular shell layout.
4. Add a focused check of the committed JSON profile's product/build/delivery
facts and absence of additional checks or credential values. Standard JSON
decoding in test code is sufficient. Keep this coverage alongside wrapper
tests; compare target membership rather than incidental list order. Avoid
reproducing upstream profile validation or remote guard tests.
5. Keep `scripts/build-release.sh` until Stage 3 removes its CI caller. The new
wrapper already uses ReleaseKit's selected build specification.
**Verification and completion:** Run `go test ./scripts`, then
`bash scripts/release.sh version` to verify real tool bootstrap and execution.
This prints the tool's version, not Audita's. Check ignored build output and
`git diff --check`, then commit. No publication or delivery invocation against
an external service is needed.
**Completed 2026-10-07:** Added the Audita profile and replaced the local
publication workflow with a wrapper that builds and execs ReleaseKit. Replaced
the publication test matrix with focused wrapper and profile checks. Checks
passed: `go test ./scripts`, `bash scripts/release.sh version`, ignored-output
verification for `bin/releasekit`, and `git diff --check`.
## Stage 3 — Migrate Woodpecker Delivery and Retire the Builder [x]
**Primary files:** `.woodpecker/release.yml` and deletion of
`scripts/build-release.sh`.
1. Adapt the maintained upstream
[tag-delivery template](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.1.0/examples/woodpecker/tag-delivery.yml).
Use its matrix and single `deliver` step with `GO_IMAGE: golang:1.25.5` and
`RELEASEKIT_VERSION: v0.1.0`. Select tag events under `refs/tags/v*`.
2. Map the command environment variable `RELEASEKIT_GITEA_TOKEN` from the
existing Woodpecker secret `GITEA_RELEASE_TOKEN`. Use the existing credential;
no new secret name or credential provisioning is selected by this migration.
3. Use the template's command, preserving Woodpecker interpolation escaping:
```sh
go run gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit@${RELEASEKIT_VERSION} ci-deliver \
--repo "$${CI_WORKSPACE}" \
--commit "$${CI_COMMIT_SHA}" \
"$${CI_COMMIT_TAG}"
```
ReleaseKit owns fetch/checkout verification, dependency preparation, builds,
note-page creation, manifests, and uploads. Remove the separate build step,
release plugin, wildcard asset settings, and local checksum configuration.
Managed delivery gets its targets and destination from `releasekit.json`.
4. Delete `scripts/build-release.sh` after confirming its active caller is gone.
Search remaining script/CI references and remove only obsolete active
invocations. Historical release records retain their original descriptions.
5. Review the parsed workflow against the template: selected image/pin, tag
trigger, token mapping, and event arguments. Use the existing YAML dependency
or available tooling for local syntax inspection. Confirm the tool pin matches
the selected module. This is adoption validation; it does not require a new
permanent workflow-text snapshot test or a running Woodpecker instance.
**Verification and completion:** Run `go test ./scripts`, review the workflow
and profile together, and run `git diff --check`. Commit the delivery migration.
The existing Go image must provide compatible Go and Git; real runner execution
is observed during subsequent delivery, independently of source completion.
**Completed 2026-10-07:** Migrated Woodpecker to the pinned managed
`ci-deliver` step using the existing secret, then removed the obsolete binary
builder. Verified the parsed workflow settings and command against the pinned
template, confirmed the module/workflow pins match, ran `go test ./scripts`,
confirmed no active CI or script caller remained, and passed `git diff --check`.
## Stage 4 — Align Policy, Documentation, and Attribution [x]
**Primary files:** `docs/release.md`, `docs/policy/release.md`, `docs/cli.md`,
`docs/internal/cli-config.md`, `docs/adr/README.md`, and
`docs/oss/attribution.md`. Update other current references only where affected.
1. Rewrite the runbook around dependency preparation, a committed candidate,
`git push origin main`, and `bash scripts/release.sh publish VERSION`.
Explain that remote `main` must identify the captured candidate exactly.
`bash scripts/release.sh check VERSION` is an optional local diagnostic,
never a required pre-publication step. Document preparation with
`GOWORK=off GOTOOLCHAIN=local go mod download`; if it changes module files,
those changes belong in the candidate before publication.
2. Describe annotated notes-derived tags, native publication statuses, and
recovery against the captured authoritative endpoint. State that the tool
retains a local tag after push failure. An exact-object recovery can confirm
success; an unreadable result remains unknown. Link to pinned upstream
recovery guidance instead of implementing or repeating its full algorithm.
Remove the old atomic-push prerequisite and procedure.
3. Describe the shared tagged-source delivery command, the existing secret
mapping, Linux artifacts, note body/footer, and separate delivery status.
Explain content-aware retries and retained-artifact recovery by linking to
upstream. Remove references to the retired builder/plugin and the assertion
that Audita has no version command. Keep the release script's bootstrap
preparation distinct from its offline candidate validation.
4. In the release policy, retain upstream wording wherever it already permits
the selected approach. Add a concise project-specific native-host check
provision under supported platforms/candidate validation: Linux delivery
targets remain required, macOS development remains best effort, and running
publication on a nonselected host requires ReleaseKit's temporary native
version check to succeed. Qualify the existing statement that macOS failures
never block release accordingly. Mac access and native tests on every target
remain unnecessary. Date the local policy update and preserve its upstream
source attribution. Avoid unrelated policy changes or additional gates.
5. Document `audita version`, its stdout/help/usage behavior, and public version
fallback in `docs/cli.md`. Add a short implementation/change-recipe note in
`docs/internal/cli-config.md` linking to that contract and the accepted ADR.
Update the ADR index to list ADR 0001; its current assertion that no ADR exists
is obsolete. Preserve the accepted ADR's content.
6. Add a House Format ReleaseKit entry with selected module v0.1.0,
`Copyright (C) 2026 Eric Rakestraw`, and `LGPL-3.0-or-later`. Link existing
`licenses/LGPL-3.0.txt` and `licenses/GPL-3.0.txt`, and the upstream COPYRIGHT
and source at commit `5e60fe3113c69f95c42fbbe32a8ca4b51f5b83c1`. Check the
selected module's notices. Keep the application grant and unchanged license
texts intact; no policy-template material is copied in this feature.
7. Repair current navigation affected by replacing this implementation plan.
Completed PromptKit/ConfigKit roadmap references that intend to identify
their former plan should link to the corresponding historical completion
records, not this ReleaseKit plan. Preserve published release notes and tags.
The next release note is deferred to release preparation.
**Verification and completion:** Check commands against the new wrapper/CLI,
local links and anchors, pinned upstream references, source attribution, and
`git diff --check`. Use the prior successful code checks for unchanged code.
Commit the documentation unit.
**Completed 2026-10-07:** Updated the ReleaseKit runbook, release policy,
version contract and contributor guidance, ADR index, and component attribution.
Repaired PromptKit and ConfigKit links to their historical implementation
records. Verified local links and anchors, release CLI and version commands,
pinned module/notices, source attribution, and `git diff --check`.
## Stage 5 — Verify Adoption and Close the Plan [x]
**Primary files:** This plan and the ReleaseKit feature roadmap; fix an
implementation file only if the final checks expose a real integration defect.
1. Audit the roadmap acceptance criteria against the landed files. Confirm the
runtime uses the public version accessor, the profile selects only the two
Linux targets, pins agree, the wrapper owns no common release mechanics, and
the old builder and repeated publication tests have been retired.
2. Prepare dependencies if necessary, then run `go test ./...` and
`go build -o ./bin/audita ./cmd/audita`. Run the normal binary's `version`
command without application setup. These tests remain offline; preparation
may use the module network separately.
3. On the committed implementation, verify the real shared builder once using
temporary output outside the repository:
```sh
releasekit_check_dir=$(mktemp -d)
bash scripts/release.sh build --output "$releasekit_check_dir" v0.0.0
```
`v0.0.0` is solely a build-test identifier, not an Audita release selection.
`build` requires no release note or remote publication. Confirm both
`audita-v0.0.0-linux-amd64` and `audita-v0.0.0-linux-arm64` are produced;
ReleaseKit's own mandatory native smoke check must report success. It checks
`audita v0.0.0` with a newline on a host-runnable executable, building a
temporary native executable itself when needed. Do not execute a foreign
architecture binary. Remove the temporary output afterward. The CLI's private
build receipt is an upstream artifact and should not enter Git.
4. Review the dependency/attribution diff, wrapper/profile wiring evidence,
workflow syntax and pin, and current documentation. Run `git diff --check`
and confirm no generated output, credentials, or unrelated work is staged.
Fix proven defects and rerun only affected checks. No real tag publication,
hosted upload, paid inference, extra platform matrix, or CI polling is needed
to establish this implementation's completion.
5. Mark the plan and feature roadmap complete after these requirements pass.
Record the tested implementation revision and checks; distinguish a later
documentation-only closure commit. Summarize the local infrastructure removed
and any remaining deployment observations accurately. Commit the closure.
The first adopting release and live delivery remain separate authorized work.
**End state:** Audita supplies a product profile, native version command, thin
local entry point, and shared CI invocation. ReleaseKit owns the common release
mechanics and their tests. Documentation and attribution match the integration.
No implementation-blocking open questions remain.
**Completed 2026-10-07:** Audited the adoption criteria against implementation
revision `5ac704c4c42fd23ed94b1f7351491ad580726db1`. Checks passed:
`go test ./...`; `go build -o ./bin/audita ./cmd/audita` followed by the binary's
`version` and `--help` commands; ReleaseKit's real `build` command produced
`audita-v0.0.0-linux-amd64` and `audita-v0.0.0-linux-arm64` and passed its
native version smoke check; profile, module/tool pins, wrapper, and parsed
Woodpecker workflow matched the selected integration; and `git diff --check`.
The closure record is a documentation-only commit after the tested
implementation revision. Audita's local build and publication mechanics and
their repeated-mechanics test matrix have been removed in favor of ReleaseKit.
The first adopting release and live Woodpecker/Gitea delivery observation
remain separate work.

View File

@@ -0,0 +1,103 @@
# Prompt Quality Roadmap
**Updated:** 2026-11-11
**Status:** Topic 1 resolved in ADR 0003; topics 2–6 remain for discussion.
Work through the remaining topics in order. Their recommendations are starting
points for discussion, not newly adopted restrictions or guarantees.
## Baseline and Intended Direction
The review of revision `6d34602` covered all eleven
[prompt definitions](../internal/prompts.md), their shared instructions, and
the context and response handling for proposals, validation, and repair.
It was a design review, not a measurement of live-model quality.
Retain the strengths: conservative transcript editing, explicit literal-span and
deletion semantics, distinct glossary context and protection, full before/after
validation, independent repair conversations with withdrawal, and native
PromptKit/ConfigKit prompt replacement. Improve consistency across the three
roles, useful rejection feedback, and maintainability.
## Discussion and Work Order
### 1. Module Focus and Editorial Acceptance — Resolved
The intentional distinction between requested module focus and acceptance of
minor editorial cleanup is recorded in
[ADR 0003](../adr/0003-module-focus-and-editorial-acceptance.md). The maintainer's
testing found that strict scope instructions also suppress desirable edits.
Apply the accepted distinction throughout the remaining prompt work.
### 2. Make Rejection Reasons Useful for Repair
Validator explanations now become repair feedback, but current instructions give
little guidance about producing actionable reasons.
**Recommendation:** ask for a concise description of the specific change, the
applicable criterion, and relevant evidence, within the validator's remit.
For example: removing "might" turns a tentative statement into a definite one;
preserve the speaker's uncertainty. Start with the existing `reason` field.
Discuss examples and sufficient detail before considering schema expansion.
### 3. Make Repair a Reconsideration of the Edit's Merits
Withdrawal is already explicit. Clarify when the model should use it and how it
should respond when the original evidence remains insufficient.
**Recommendation:** ask the model to reassess the correction from the supplied
evidence, retain the module's focus and the accepted editorial allowance, and
withdraw unsupported proposals. Address the risk that numerical confidence
feedback merely encourages raising the score.
Do not automatically prohibit confidence-only revisions: legitimate reassessment
is possible. Discuss and evaluate that behavior before adding a business rule.
Retain the agreed full-history conversation model and existing repair lifecycle.
### 4. Simplify Instructions and Improve Examples
Restrictions and examples recur across proposal, validation, and repair assets;
the grammar prompt repeats its word-substitution boundary several ways.
**Recommendation:** distinguish shared editing principles, requested module focus,
validator acceptance criteria, and task-specific response contracts. Use small
shared fragments where they reduce drift. Replace some repetitive example lists
with contextual contrasts, such as
accidental repetition versus deliberate emphasis. Preserve useful distinctions;
shorter prompts alone are not the objective. Discuss what belongs in shared
fragments and what remains local to each definition.
### 5. Evaluate Editorial Quality
Rendering, schema, and controlled-response tests establish integration behavior,
but do not establish the quality of model editorial judgments.
**Recommendation:** assemble a modest evaluation set containing both needed edits
and text that should remain unchanged. Include ambiguous glossary matches,
intentional repetition, meaningful uses of apparent filler, punctuation that
changes meaning, and repairs that should be withdrawn. Compare unnecessary edits,
missed corrections, validator disagreement, and repair outcomes using independent
editorial assessment; validator approval alone is insufficient evidence.
Capture candidate examples during topics 1–4 and establish a baseline before
changing prompts. Discuss representative models, expected outcomes, and how to
run evaluations. Keep model-dependent evaluation separate from the default
offline test suite; no numerical quality gate is selected here.
### 6. Revisit Broader Product Choices
Use the preceding decisions and evaluation evidence to discuss these in order:
1. **Grammar naming and scope:** the current module is close to copy editing,
with an isolated article exception. Decide whether its name and scope convey
the intended requested work, preserving the accepted editorial allowance.
2. **Glossary/homophone overlap:** assess the benefit of their overlapping remit
and repeated glossary passes before changing the default sequence.
3. **Language and examples:** decide whether spoken English and campaign-specific
examples are the right bundled defaults, and how broadly those defaults
should apply.
4. **Meaning-reversal policy:** the guard can also exclude a genuine correction
of a mistranscription. Initially retain the conservative default while
explicitly evaluating that tradeoff.
These are product discussions, not selected renames, broader editing permissions,
language-support promises, or changes to validation policy.

View File

@@ -1,276 +0,0 @@
# PromptKit v0.12.0 Adoption Roadmap
**Last updated:** 2026-10-04
**Status:** Complete. The implementation plan records the ordered work and
verification details.
## Purpose and Baseline
Adopt PromptKit v0.12.0 now that the independent
[ConfigKit v0.5.0 upgrade](configkit.md) is complete. Align Audita with the
libraries' native configuration model while retaining Audita's domain rules,
application-owned backend settings, and resilient processing workflow.
At reviewed Audita revision `8cba751835ba9faad340e933a3cfbb59c03924fd`,
[go.mod](../../go.mod) selects ConfigKit v0.5.0 and PromptKit v0.11.2, with
OpenRouter and Rakestrawhome `/v2` catalogs at `v2.20260926.0`. Go 1.25.5
remains the application minimum. The implemented contracts belong to
[configuration](../config.md), the
[LLM integration](../integrations/openai-compatible-llm.md), and the
[LLM runtime](../internal/llm-runtime.md).
`internal/framework/llm.NewRuntime` currently assembles one engine using the
old prompt/profile filesystem options and programmatic backend registration.
Audita embeds seven prompt definitions and separate response schemas. Operators
select proposal and validation profiles from `profiles_dir`; the effective
backend must be registered in application configuration, its endpoint must
match, and its credential comes from Audita's ConfigKit-resolved settings.
Successful profile reads are recorded to protect those sources from output
collisions. Both roles share the same engine's backend admission controls.
The reviewed upstream release is
[PromptKit v0.12.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/releases/v0.12.0.md),
commit `4b8169da286917fde7875425bb32b799b12c6af3`. It depends on ConfigKit
v0.5.0 and both maintained `/v3` catalog modules at `v3.20260930.0`. This is an
intentional incompatible pre-1.0 minor release: it replaces construction and
configuration APIs, rather than simply updating provider data. PromptKit's
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/formats.md)
and [consumer migration guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
own the upstream details.
## Target End State
- Audita imports released PromptKit v0.12.0 and its `/v3` catalogs, retaining
ConfigKit v0.5.0 and using no local dependency replacement.
- The runtime uses `Config.Sources` and native ConfigKit source groups.
PromptKit owns composition, inheritance, presence, final-definition
validation, prompt capture, and execution. Audita does not translate old
profiles or implement a second YAML loader or inheritance resolver.
- Embedded prompts and maintained operator profiles use keyed `prompts` and
`profiles` mappings. Existing prompt IDs, versions, role selections,
instructions, and response-schema associations remain the adoption baseline.
The application configuration schema, overlay selection, CLI interface, and
main-file-relative `profiles_dir` resolution remain in place.
- Native profile `extends`, field-presence semantics, and source-relative
resource rules apply. Documents can contain multiple definitions, and IDs
come from mapping keys rather than filenames. The existing `profiles_dir`
accepts native unified PromptKit documents containing profiles, backends,
and prompts, including intentional operator prompt overrides.
- Application backend endpoints, configured concurrency and queue capacity,
and credential ownership remain authoritative for selected roles. Preserve
the existing profile-selection and prepared-target checks; do not introduce
a second admission controller or implicit provider credential lookup.
- A runtime uses one captured configuration snapshot for both roles. Each CLI
invocation constructs a new engine; hot reload is not part of this feature.
Existing inspection remains offline and safe, and consumed host sources
remain protected from output collisions.
- PromptKit retains responsibility for transport retries, schema repair,
observation, and prepared execution. Preserve Audita's configured retry and
repair budgets, operation deadlines, cancellation, partial-success behavior,
error/recovery reporting, and debug selection.
## Configuration Assembly and Ownership
Replace removed `WithPromptFS`, `WithProfileFS`, and `WithBackend` usage with
the released source API. Discover embedded YAML through `FilesystemInputs` and
operator YAML through `DirectoryInputs`, with explicit caller tiers. Keep the
retained schema filesystem integration and HTTP client configuration.
Use the helpers' default-disabled imports; this adoption does not require new
application source-selection flags or operator underlay tiers.
PromptKit supplies catalog definitions first and its own defaults next. Caller
tiers then overlay in their supplied order, followed by an optional highest
priority main input. Files within one directory group compose strictly;
alphabetical filenames do not establish override precedence. Use separate
tiers for genuine priority differences rather than manufacturing file-order
semantics.
Represent application backend settings as a final programmatic source, using
`BackendSource` or ConfigKit mappings where explicit null is needed. Preserve
configured zero queue capacity as an explicit value. Suppress backend-derived
credential environment lookup explicitly where catalog layering would otherwise
introduce it; omission alone can retain a lower-tier value. Continue detecting
competing effective profile credential lookups rather than silently clearing
operator profile fields. A missing backend credential remains valid for an
unauthenticated local service unless the selected profile requires a key.
Catalog backend IDs are no longer reserved upstream. Audita should not invent a
new reservation rule. Catalog selection alone must not bypass application
registration, endpoint, capacity, or credential requirements. Catalog profile
contents may evolve; this upgrade does not promise fixed model choices or live
provider availability.
The accepted caller order is embedded Audita prompts, operator definitions,
then application backend settings. The existing `profiles_dir` accepts native
unified PromptKit documents containing profiles, backends, and prompts. Keep
the name and path setting for continuity, and document its broader accepted
content. This follows the upstream source API without a parallel parser or
top-level filtering. Profiles, provider defaults, and intentional prompt
overrides use the same native composition model.
Audita continues to ship its domain prompts, enforce its selected-backend
requirements, and supply its per-call schemas. Operator prompt overrides are
an accepted capability of this adoption; documentation, source accounting,
and focused behavioral coverage should reflect them.
## Work Required
### Runtime Construction and Source Accounting
Migrate `internal/framework/llm/runtime.go` to layered construction and retain
the role-bound adapter boundary. Review startup inspection and failure
classification because upstream now validates all final definitions, including
unused ones, during engine construction. Distinguish invalid operator inputs
from broken embedded assets under the existing
[CLI exit contract](../cli.md); do not treat every construction failure as an
operator error merely because validation moved earlier.
Upstream permits partial profiles and defers endpoint/model completeness until
request overrides are applied. Audita supplies no model override, so explicitly
validate that both selected role profiles have a nonblank model and a complete
effective endpoint during offline configuration validation and runtime setup.
Retain the application-backend, endpoint-match, and credential checks. Do not
rely on engine construction or profile inspection alone to establish selected
profile completeness. Unselected partial profiles remain permitted if they
satisfy upstream definition validation; they need not be runnable.
Adapt source accounting to the new discovery/capture boundary. The existing
recording filesystem wrapper may be replaceable with discovered inputs or
upstream source metadata. Protect the host sources actually consumed by the
runtime, including inherited and differently named definitions, without
guessing filenames from selected profile IDs. Account for consumed operator
message files as well. Keep this proportionate to the existing collision
contract; do not add filesystem concurrency or power-loss guarantees.
Keep the existing safe effective-profile summaries and application explanations.
Use upstream metadata where useful, but do not expose the full
`InspectConfiguration` snapshot as ordinary CLI or report output: prompt text
and arbitrary provider fields are outside those summaries' current scope.
### Prompt Assets, Profiles, and Presence Semantics
Move each of the seven files in `internal/prompts/assets/prompts` to the keyed
prompt/version format. Keep the shared instruction fragments and the complete
per-call output contracts owned by Audita. This migration does not authorize
another prompt-quality rewrite or changes to domain validators.
The adapter supplies `HardeningPolicy`, `ProposalContract`, and
`ValidationContract` as template variables. Embedded prompts retain their
references to the applicable variables. Operator message overrides control
whether those variables are rendered; supplying a variable does not inject its
text into a template that omits it. Preserve this native override behavior and
document the distinction. Per-call schemas and domain validation remain enforced
regardless of those references. Unconditional shared-instruction injection is
outside this adoption.
Convert the six maintained profiles in `examples/profiles` and test-generated
profile/prompt fixtures to the new format. Convert any old `base_profile`
usage to `extends`, and remove embedded identities. Migrate tests that construct
engines using removed options or old `Profile`/`Backend` values.
Reorganize the maintained profiles into separately selected local and service
directory roots. The current shared directory includes profiles referring to
backends absent from each example's application configuration; eager validation
would reject those unselected definitions after a format-only conversion.
Update `profiles_dir` in `examples/minimal-config.yml`,
`examples/production-config.yml`, and `examples/configkit-inheritance.yml` to
select the appropriate root. Keep the service testing profiles in the service
root so the testing overlay remains usable. Discovery is recursive: placing
both groups beneath a common selected root does not isolate them. Update example
links and validation recipes to match the final layout.
Review presence deliberately rather than mechanically preserving old serialized
values. In particular, explicit zero temperature, top-p, or max-tokens now means
an outbound zero; zero timeout disables PromptKit's generation deadline while
Audita's outer operation deadline still applies. Omission, blank values, and
null have field-specific meanings. Empty provider maps do not erase inherited
entries, and changing model/backend does not erase unrelated explicit settings.
Use the native meanings without a compatibility shim. Retaining the examples'
literal `temperature: 0` as an explicit provider setting is recommended; explain
that omitting the field requests an unspecified value, and null suppresses an
inherited control. Do not automatically turn every existing zero into omission
or every omission into a zero. Demonstrate representative profile inheritance
without requiring every maintained example to use it.
### Documentation, Migration, and Attribution
Update the profile section of `docs/config.md` and maintained examples when the
implementation lands. Explain the broader accepted content of `profiles_dir`,
operator prompt overrides, their precedence and optional use of shared instruction
variables, keyed identities, composition versus precedence, profile inheritance,
presence, eager final-definition validation, selected-root errors, and captured
configuration. Link to the pinned
upstream reference for generic field semantics rather than reproducing its
entire contract.
Include operator migration guidance for old profile documents and behaviorally
significant zeros/nulls. This is a deliberate format migration, not an automatic
upgrade of existing installation files. Preserve supported application overlays,
including the cheaper/faster testing-profile workflow.
Update the pinned consumer link in `docs/integrations/openai-compatible-llm.md`,
construction/source-accounting descriptions in `docs/internal/llm-runtime.md`,
and asset-format guidance in `docs/internal/prompts.md`. Update other affected
references only where implemented behavior changes.
Update `go.mod`, `go.sum`, and `docs/oss/attribution.md` for PromptKit and the two
catalog modules. Verify exact selected source revisions and upstream notices
under the [OSS policy](../policy/oss.md); retain the applicable LGPL/GPL and MIT
texts rather than duplicating unchanged license documents. Dependency adoption
does not change Audita's application license.
Plan this adoption for an Audita major release under the
[release policy](../policy/release.md#versioning-and-compatibility). The public
profile format changes incompatibly and no compatibility loader is planned;
PromptKit's own pre-1.0 minor version does not change Audita's classification.
The exact version and publication timing remain deferred to release preparation.
## Verification and Completion
Use existing offline tests as the starting point. Update fixtures intentionally,
and add focused behavioral coverage where the source migration creates a
distinct risk. Completion means:
- The released dependency set builds and the complete Audita test suite passes.
- All registered embedded prompts load and render from keyed assets, retaining
their applicable shared-instruction references and established per-call schemas.
- Offline configuration validation rejects incomplete selected profiles, including
a missing or blank model, while accepting valid unselected partial profiles.
- Selected roles use final application backend settings and direct resolved
credentials; both roles still share configured capacity, including explicit
zero queue capacity. Catalog layering does not introduce a competing lookup.
- Representative keyed profiles, native inheritance, and explicit presence work
through the real runtime. Directory conflicts and invalid final definitions
produce useful startup errors without filename-derived identity assumptions.
- Operator prompt and backend contributions follow the accepted source order;
prompt overrides render, including when their messages omit shared-instruction
variables, while Audita retains its per-call schemas, domain validation, and
authoritative application backend settings.
- Source collision protection, safe inspection, and operator-versus-asset error
classification remain effective at the changed construction boundary.
- Representative prepared execution, schema repair, transient retry, and
partial-success tests retain their existing behavior. Reuse adequate coverage
rather than duplicating PromptKit's complete upstream test matrix.
- Local, service, testing-overlay, and ConfigKit inheritance examples validate
offline using their separately selected profile roots. Canonical documentation
describes the migrated format, shared-instruction behavior, and major-release
migration; attribution identifies the selected library sources accurately.
No live provider calls, new release gates, or CI-success dependency are needed
to establish adoption. Guided domain recovery and the other work recorded in
[future.md](future.md) remain separate features. Do not expand this upgrade into
new report formats, pipeline scheduling, or general configuration-inspection
commands. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/418da456d75a68d3bb5a43ab823c5b2b434b90cc/docs/roadmap/implementation.md)
records that work's execution steps and verification.
## Completion Record
Implementation code is recorded through revision `5525fe8`; documentation was
updated in `9bcf183`, and the final roadmap closure is documentation-only. At
`9bcf183`, `go build -o ./bin/audita ./cmd/audita`, `go test ./...`, all six
offline maintained-example commands in the implementation plan, `go mod
verify`, and `git diff --check` passed. The selected dependency graph and
commit-pinned attribution match PromptKit v0.12.0 and both `/v3` catalogs at
`v3.20260930.0`.

View File

@@ -1,216 +0,0 @@
# ReleaseKit Adoption Roadmap
**Last updated:** 2026-10-07
**Status:** Implementation complete; first adopting release and live delivery
observation pending.
## Purpose and Original Baseline
Implement the decision in [ADR 0001](../adr/0001-releasekit.md): standardize
Audita's release process on ReleaseKit to improve maintainability and reduce
locally owned release-focused code. Those benefits outweigh the small advantages
of retaining Audita's custom procedure. The workflow changes required by the
shared standard are accepted.
At Audita revision `6de470b53d185b10327ec4deba23f5a84e606e1f`, v3.0.0 is
released. Audita owns `scripts/release.sh`, `scripts/build-release.sh`, and
`scripts/release_test.go`, totaling 207 lines, plus the 26-line Woodpecker delivery
workflow. Publication validates a clean candidate, cross-builds Linux binaries,
and atomically pushes `main` and a lightweight tag. Woodpecker builds artifacts
and uses a release plugin to publish the note, binaries, and checksums. Audita
has no runtime version command.
The adoption baseline is ReleaseKit v0.1.0, commit
`5e60fe3113c69f95c42fbbe32a8ca4b51f5b83c1`. Its public library, executable,
`buildinfo` package, and maintained Woodpecker integration cover the required
mechanics. The upstream test suite passes; its documentation records live
Woodpecker/Gitea verification as remaining deployment work.
## Current Baseline
The adoption implementation is now present: `go.mod` selects ReleaseKit
v0.1.0, `releasekit.json` supplies Audita's product and delivery facts,
`scripts/release.sh` builds and forwards arguments to its executable, and
`.woodpecker/release.yml` invokes the same pinned version for tag delivery.
The application exposes `audita version` through `buildinfo`; the custom
release builder has been removed. The sections below retain the intended
adoption scope rather than describing work that still needs implementation.
The [adoption review](remediation.md) records focused CLI and wiring-test
findings and their resolution status. Remaining implementation corrections
belong there. First-release delivery observation remains deployment work,
separate from source-release completion.
## Target End State
- ReleaseKit owns candidate validation, source publication, version injection,
selected-target builds, checksum generation, and hosted delivery. Audita owns
its product facts, compatibility decisions, release notes, and necessary
product-specific checks.
- A small `releasekit.json` declares product `Audita`, remote `origin`, main
package `./cmd/audita`, binary name `audita`, and delivery targets
`linux/amd64` and `linux/arm64`. Its Gitea destination identifies
`https://gitea.maximumdirect.net`, owner `eric`, and repository `audita`.
- Maintainers commit the candidate and publish it to authoritative `main`
before invoking ReleaseKit publication. ReleaseKit validates that candidate,
creates an annotated tag from the committed note, and pushes its exact tag
object. Confirmed source publication completes the release independently of
CI and artifact delivery.
- One captured candidate receives ReleaseKit's standard offline test suite and
selected build verification. Audita's profile introduces no additional checks
without a concrete product need. The optional local check remains a diagnostic
operation rather than a prerequisite that duplicates publication validation.
- `audita version` reports `audita <version>` through ReleaseKit's public
`buildinfo.Version()` accessor. Ordinary development builds use its native
fallback behavior, including `dev` when release information is unavailable.
- Woodpecker invokes the pinned shared `ci-deliver` command. Hosted notes,
Linux assets, and `SHA256SUMS` retain their current identities; delivery
compares existing content, adds missing assets, and reports conflicts and
uncertainty separately from source publication.
- Audita's release code consists of thin tool invocation and product wiring,
with focused integration coverage. Common mechanics and their repeated tests
reside upstream.
## Accepted Standardization Choices
Adopt ReleaseKit's main-first publication and annotated tags, replacing the
atomic branch/tag push and lightweight tags for new releases. Existing released
tags, notes, pages, and assets retain their historical contents.
Adopt its standard build settings and mandatory native version smoke check.
The check verifies that version injection reaches the executable. On a host
outside the selected Linux targets, ReleaseKit can build a temporary native
binary; publication from that host depends on that check succeeding. This is an
accepted workflow change, including for macOS invocations. Linux remains the
production delivery target, and macOS remains a best-effort development platform.
Mac access and macOS artifact delivery are not release prerequisites.
Use the shared managed build-and-deliver operation for ordinary CI delivery.
ReleaseKit's retained-artifact workflow remains available when a delivery retry
needs the original bytes. Follow its content-aware recovery behavior rather
than reproducing that machinery in Audita.
## Adoption Scope
### Dependency, Profile, and Tool Entry Point
Select the released v0.1.0 module in `go.mod` and `go.sum`. The application uses
the lightweight `buildinfo` facade; release orchestration is exercised through
the upstream executable. The sibling checkout is review material, not a
production dependency replacement.
Add the product profile described above. Leave additional checks empty and keep
delivery tokens out of the profile. Use `RELEASEKIT_GITEA_TOKEN` as the token
environment variable, supplied from the existing Woodpecker
`GITEA_RELEASE_TOKEN` secret.
Reduce `scripts/release.sh` to repository selection, building the CLI selected
by Audita's module graph into ignored `bin/` output, and invoking it directly.
Forward ReleaseKit's native commands and arguments and preserve its exit status,
including uncertain publication status. Use the same released tool version in
the module graph and CI pin. Dependency/tool preparation belongs before offline
candidate validation.
### Runtime Version Reporting
Add `version` to the existing CLI dispatch and help, with output
`audita <version>` followed by a newline and exit status zero. This operation
works without loading application configuration, resolving LLM credentials, or
contacting a provider. Use the public accessor and let ReleaseKit own injection
mechanics. Keep the release orchestration dependency outside transcript
processing.
### Publication and Binary Delivery
Replace the local publication and cross-build mechanics with ReleaseKit's
commands and profile. Retire `scripts/build-release.sh` when its callers use
the shared builder.
Replace the current Woodpecker build/plugin sequence with the maintained
tag-delivery integration, pinned to v0.1.0. Retain a compatible Go 1.25.5 image
with Git, the existing secret, and only the selected Linux targets. Pass the
workspace, event commit, and version to `ci-deliver`; the shared command owns
tagged-checkout verification, dependency preparation, build, and delivery.
The declared artifacts remain `audita-<version>-linux-amd64`,
`audita-<version>-linux-arm64`, and `SHA256SUMS`. The committed release note
supplies the hosted body and required licensing footer. Delivery status and
recovery follow the upstream operation; source publication does not poll CI or
wait for assets.
### Tests and Removal of Duplicated Mechanics
Replace the repeated release-mechanics matrix in `scripts/release_test.go`
with proportionate coverage of Audita's wiring. Protect the new version command
and wrapper/profile integration where these introduce distinct consumer risks.
Verify actual version injection and the selected Linux builds through the real
builder during adoption validation. Use local Git/HTTP fixtures and controlled
command boundaries where needed; keep the default suite offline.
Reuse ReleaseKit's coverage for common Git guards, push reconciliation,
checksums, and delivery retries. Evaluate the reduction in local machinery from
the resulting implementation rather than imposing a numerical deletion target.
### Documentation and Attribution
Update `docs/release.md` for main-first preparation, native ReleaseKit commands,
annotated tags, dependency preparation, delivery status, and failure recovery.
Align `docs/policy/release.md` where necessary to describe the deliberately
accepted native-host check while retaining the existing production targets and
independent source-completion rule.
Update `docs/cli.md`, relevant internal CLI guidance, and the ADR index for the
version command and accepted decision. Keep the ADR's decision content intact.
Record the adoption and maintainer workflow change in the next release note
when that release is prepared.
Add ReleaseKit's selected source and LGPL-3.0-or-later notices to
`docs/oss/attribution.md` under the existing OSS policy. Reuse applicable license
texts already present. Incorporating `buildinfo` does not change Audita's
application license.
## Verification and Completion
Adoption is complete when:
- The released dependency and pinned tool are selected consistently, and the
application builds with its default offline test suite passing.
- Version output works without LLM setup and reports the injected version from
a ReleaseKit-built executable.
- The native profile and thin wrapper drive candidate checks and publication
without Audita retaining common publication or build mechanics.
- Both required Linux targets build through ReleaseKit, and the CI workflow
declares the intended destination, artifacts, note body, and secret mapping.
- Focused consumer coverage exercises the changed boundaries, and redundant
mechanics tests and the old builder have been retired.
- Current documentation and attribution describe the adopted behavior.
## Remaining Release and Deployment Work
Prepare the first adopting release under the [release procedure](../release.md),
including a release note explaining the maintainer workflow change. Resolve
outstanding adoption-review findings through the linked remediation record.
Version selection and publication timing remain release-preparation decisions.
Offline integration evidence and live delivery observations are distinct.
Observe the shared workflow during the first adopting release and address any
deployment failures through ReleaseKit's delivery recovery. Check that the
tagged commit, hosted note with its licensing footer, selected Linux assets,
and checksums reach the intended destination. This observation is asynchronous;
CI success or asset availability does not become a source-release completion
gate or require the publication command to wait.
The accepted ADR settles the architectural choice and standard workflow
trade-offs. No significant feature decisions remain open. The next Audita
version and publication timing belong to release preparation. The
[implementation plan](implementation.md) owns the ordered execution steps and
their verification; this roadmap owns the desired end state and adoption intent.
Implementation completed 2026-10-07 at revision
`5ac704c4c42fd23ed94b1f7351491ad580726db1`. The full Go suite and executable
version path passed, and ReleaseKit built both selected Linux artifacts with
its native version smoke check passing. The profile, dependency and CI pins,
wrapper, and parsed workflow were verified together. Audita's local release
builder and publication mechanics are retired. This completion record is a
later documentation-only commit; live delivery remains to be observed during
the first adopting release and is separate from source implementation.

View File

@@ -1,151 +0,0 @@
# ReleaseKit Adoption Review: Findings and Recommended Changes
**Reviewed:** 2026-10-07, source commit
`3c405f0524ec6c098d00d4e371abbace7648d446`.
**Scope:** ReleaseKit adoption commits after `721365e`, including CLI version
reporting, local release wiring, CI delivery, tests, and documentation.
**Status:** R1–R3 addressed. The findings below describe the reviewed
baseline; completed work is recorded under Resolution Record.
## Architecture Assessment
The core adoption matches [ADR 0001](../adr/0001-releasekit.md) and the
[feature roadmap](releasekit.md). Audita supplies product facts in
`releasekit.json`, a thin executable wrapper, a configuration-free version
command using the public `buildinfo` accessor, and a pinned Woodpecker
`ci-deliver` invocation. ReleaseKit owns candidate validation, annotated-tag
publication, version injection, Linux builds, checksums, and hosted delivery.
The custom builder and local publication mechanics have been retired.
No core ownership or delivery-target gap was found. The findings concern one
CLI behavior defect, an ineffective target-selection assertion, and unnecessary
test restrictions. Main-first publication and the native-host version smoke
check are deliberately accepted workflow changes, not accidental strengthening.
## R1 — P2: Simplify Version Help Handling and Correct Its Streams
**Category:** CLI behavior defect and unnecessary argument restriction.
**Evidence:** [runVersion](../../internal/cli/run.go) handles standalone `-h`
and `--help` before parsing, then rejects either spelling when combined with
other arguments. This adds a special prohibition beyond the requested usual
help behavior. For example, `audita version -h --help` returns status `2`.
The parser's help path also writes usage twice: `FlagSet.Usage` writes to stderr,
and the `flag.ErrHelp` branch writes it again to stdout. Reproduced with
`audita version --h` and `audita version -help`: both return status `0` and emit
the same usage on both streams. The standard library recognizes these help
spellings, but the preliminary special cases do not handle them.
**Recommended change:** Use one standard parsing path. Let help short-circuit
according to the parser's ordinary behavior, and emit successful help once to
stdout with empty stderr. Keep non-help unknown flags and unexpected positional
arguments as usage errors. Remove the bespoke mixed-help prohibition and the
test that promotes it into a requirement. Update the CLI wording if needed so
it describes ordinary help handling without promising validation of trailing
arguments after a help request.
**Acceptance:** Normal version output remains configuration-free and unchanged.
Representative parser help paths produce status `0`, stdout usage, and empty
stderr. Non-help invalid invocations still return status `2`. Tests protect
streams and meaningful behavior without establishing a new argument-combination
policy or a comprehensive help-alias contract.
## R2 — P2: Make the Delivery-Target Check Compare Actual Membership
**Category:** Test correctness gap.
**Evidence:** [sameMembers](../../scripts/release_test.go) compares slice
lengths and checks only that each observed item belongs to the expected slice.
It never checks that every required target is present. Calling the actual helper
with `got = [linux/amd64, linux/amd64]` and
`want = [linux/amd64, linux/arm64]` returns true. A profile missing arm64 can
therefore pass the selected-target assertion. ReleaseKit deduplicates repeated
targets, so that configuration would build only amd64.
The reverse problem also occurs: a list containing both required targets plus
a repeated amd64 entry returns false despite having the same effective target
membership. The current committed profile is correct; the defect is in its
regression protection.
**Recommended change:** Compare the sets of configured and required targets,
including coverage of every required member and rejection of an unintended
member. Reuse straightforward standard-library data structures. Duplicate
normalization already belongs to ReleaseKit; Audita needs no duplicate ban or
order requirement.
**Acceptance:** A repeated single target cannot conceal an omitted production
target. Either omitted required target and an additional unintended target fail
the profile check. Reordered or repeated entries with the same effective target
set remain equivalent. Focused coverage catches the demonstrated false positive.
## R3 — P3: Remove Incidental Wiring-Test Requirements
**Category:** Tests strengthen representation choices into contracts.
**Evidence:** [release wrapper/profile tests](../../scripts/release_test.go)
require the exact output argument spelling `-o ./bin/releasekit` and reject the
presence of a `checks` key regardless of its value. In disposable copies of the
tests, changing the output path to the equivalent `bin/releasekit` fails the
wrapper assertion even though execution still reaches the intended tool.
Adding `"checks": []` fails the profile assertion despite configuring no
additional commands.
The accepted requirements concern the executable destination, native argument
and status forwarding, and avoiding redundant additional checks. They do not
require one path spelling or one serialization of an empty list. The plan's
instruction to omit `checks` selects an initial file representation rather than
an enduring prohibition on equivalent configuration.
**Recommended change:** Check the resolved output destination and observed
wrapper behavior instead of the literal relative-path spelling. Check that the
effective additional-check list is empty, accepting omission or an empty list.
Keep the existing product, target, destination, argument-boundary, failure, and
exit-status protection. Simplify fixture helpers where that reduces maintenance
without adding another parser or test framework.
**Acceptance:** The two demonstrated equivalent edits preserve passing wiring
tests, while a wrong executable destination, changed product facts, nonempty
additional checks, lost argument boundaries, or lost tool status still fail.
The tests remain offline and do not duplicate ReleaseKit's release-mechanics
matrix.
## Review Verification and Limits
- `go test ./...` passed on the reviewed source.
- A normal application build succeeded. Direct executable invocations reproduced
the version-help stream defect and mixed-help rejection.
- The actual membership helper reproduced the missing-target false positive.
Disposable test copies reproduced both equivalent-configuration failures;
the repository's code and tests were left unchanged.
- The real ReleaseKit builder produced both configured Linux artifacts with
build-test identifier `v0.0.0` and passed its injected-version smoke check.
This used the available Go 1.26.8 Linux amd64 toolchain; CI remains configured
for Go 1.25.5. Temporary build artifacts were removed.
- Module selection, source attribution, product profile, secret mapping, and
CI command match the intended integration. Live Woodpecker/Gitea delivery
remains unobserved, as already recorded in the feature roadmap; it is separate
deployment work rather than an implementation finding or source-release gate.
- No tags, pushes, hosted uploads, or LLM calls were made. Remediation should
remain bounded to the findings above and add no new release gates.
## Resolution Record
- **R1:** The version command now uses one standard flag-parsing path. Successful
help is written once to stdout, with no stderr usage, and help can short-circuit
trailing arguments. Non-help usage errors retain status `2`. Regression tests
reproduced both defects before the fix; `go test ./internal/cli ./cmd/audita`
passed afterward. The CLI contract reflects ordinary help behavior.
- **R2:** Delivery targets are compared as sets using the standard library.
Regression cases reject either missing Linux target concealed by duplicates
and an unintended target, while accepting equivalent reordered/repeated lists.
The cases reproduced the faulty membership behavior before the fix;
`go test ./scripts` passed afterward.
- **R3:** Wrapper coverage now checks the executable's actual destination, and
profile coverage checks the effective additional-check list. Equivalent output
paths and an explicit empty list are accepted. Fixture helpers use standard
library operations where appropriate. Disposable mutation checks accepted
`-o bin/releasekit`, `-o=bin/releasekit`, and `"checks": []`, while detecting a
wrong executable destination, changed product, nonempty checks, lost argument
boundaries, and lost tool status. The complete `go test ./...` suite passed
with all three remediations in place. No new release gates were added.

82
examples/README.md Normal file
View File

@@ -0,0 +1,82 @@
# Maintained examples
Build Audita from the repository root with Go 1.27 or newer. The commands below
assume the executable is on `PATH`. Each project selects its main file from its
own directory, which also determines companion resources and overlay context.
All models, credentials, names, and endpoints are illustrative.
## Local processing
From `examples/local`, edit `prompts.yml` to match a reachable OpenAI-compatible
local service and its model. Then run:
```sh
audita config validate --config config.yml
audita process --config config.yml --output corrected.json transcript.json
```
Processing makes model requests. Reports are written beside the selected output
unless disabled. For an offline validator preview, from the same directory:
```sh
audita promptkit show prepared audita/validators.meaning_reversal_review \
--config config.yml --profile local \
--var "PayloadJSON=$(cat validation-candidate.json)" \
--var 'TranscriptDescriptionBlock='
```
To preview the repair instruction and its output schema without inference:
```sh
audita promptkit show prepared audita/modules.grammar.repair \
--config config.yml --profile local \
--var 'RepairRequestJSON={"proposal_ref":1,"correction":{"id":1,"original_text":" ","corrected_text":"","confidence":0.9},"findings":[]}'
```
These previews render supplied data without inference. The native command help
and [CLI reference](../docs/cli.md) describe additional inspection and validation.
## Service and overlay
From `examples/service`, edit the synthetic endpoint and models in `prompts.yml`.
Set `AUDITA_EXAMPLE_KEY` to your service credential before a real request. Inspect
without a provider call:
```sh
audita config validate --config config.yml --overlay testing
audita promptkit show profile service-test --config config.yml
```
The example does not promise a provider's model availability, price, or quality.
## Edit a prompt and run again
From `examples/prompt-edit`, inspect the operator replacement:
```sh
audita promptkit show prompt audita/modules.grammar.proposal --config config.yml
```
Edit its message in `prompts.yml`, repeat the inspection, and run `audita process
--config config.yml --output corrected.json transcript.json` against a configured
local service. Each invocation captures the files anew.
## Shared glossary library
From `examples/campaigns`, `first.yml` and `second.yml` each consume the common
`shared.yml` through the companion `glossary.yml`. Inspect each configuration:
```sh
audita config validate --config first.yml
audita config validate --config second.yml
```
With a local service configured, process either campaign:
```sh
audita process --config first.yml --output first-output.json transcript.json
audita process --config second.yml --output second-output.json transcript.json
```
Edit `prompts.yml` for a local service before processing either campaign. The
first uses separate party and spell corrections; the second selects only party.

View File

@@ -0,0 +1,8 @@
version: 3
llm: {proposal_profile: local}
glossaries: [party, spells]
pipeline:
stages:
- {id: names, module: glossary, glossaries: [party]}
- {id: magic, module: glossary, glossaries: [spells]}
- {module: grammar}

View File

@@ -0,0 +1,4 @@
config:
imports:
- path: shared.yml
required: true

View File

@@ -0,0 +1,5 @@
profiles:
- id: local
endpoint: http://localhost:11434/v1
model: example-model
api_key_required: false

View File

@@ -0,0 +1,7 @@
version: 3
llm: {proposal_profile: local}
glossaries: [party]
pipeline:
stages:
- {id: names, module: glossary}
- {module: grammar}

View File

@@ -0,0 +1,12 @@
party:
entries:
- id: lyra
name: Lyra
category: character
description: A fictional adventurer.
spells:
entries:
- id: shield
name: Shield
category: spell
aliases: [Ward]

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Narrator",
"start": 0,
"end": 2,
"text": "Lyra raises a ward before the journey."
}
]

View File

@@ -1,15 +0,0 @@
version: 2
profiles_dir: profiles/local
backends:
local-base:
endpoint: http://127.0.0.1:11434/v1
concurrency: 4
queue_capacity: 0
local:
extends: local-base
concurrency: 2
llm:
proposal_profile: local-proposal
validation_profile: local-validation
pipeline:
modules: [grammar]

View File

@@ -0,0 +1,8 @@
version: 3
llm:
proposal_profile: local
pipeline:
stages:
- module: glossary
- module: grammar
glossaries: [party]

View File

@@ -0,0 +1,7 @@
party:
entries:
- id: jesters
name: Jesters
category: faction
description: A traveling group of performers.
aliases: [Jester]

View File

@@ -0,0 +1,5 @@
profiles:
- id: local
endpoint: http://localhost:11434/v1
model: example-model
api_key_required: false

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Narrator",
"start": 0,
"end": 2,
"text": "The gestures gather beside the stage."
}
]

View File

@@ -0,0 +1 @@
[{"correction_index":1,"original_text":"not open","corrected_text":"open","original_segment_text":"The door is not open.","corrected_segment_text":"The door is open."}]

View File

@@ -1,13 +0,0 @@
version: 2
profiles_dir: profiles/local
backends:
local:
endpoint: http://127.0.0.1:11434/v1
concurrency: 1
llm:
proposal_profile: local-proposal
validation_profile: local-validation
pipeline:
modules: [grammar]
output:
schema: bare-segments

View File

@@ -1,3 +0,0 @@
llm:
proposal_profile: service-proposal-testing
validation_profile: service-validation-testing

View File

@@ -1,41 +0,0 @@
version: 2
profiles_dir: profiles/service
backends:
service:
endpoint: https://openrouter.ai/api/v1
credential: AUDITA_SERVICE_KEY
concurrency: 2
queue_capacity: 1024
llm:
proposal_profile: service-proposal
validation_profile: service-validation
transient_retries: 2
repair_attempts: 2
operation_timeout: 15m
pipeline:
modules: [glossary, homophones, glossary, spoken_word, grammar]
output:
schema: audita-v1
chunking:
# Requires at least this many normalized segments; omit for automatic chunking.
target_sections: 8
max_section_tokens: 8192
min_section_tokens: 2048
normalization:
max_segment_gap: 4.0
ellipsis_gap: 3.5
max_segment_duration: 60.0
max_segment_tokens: 2048
thresholds:
glossary: 0.8
homophones: 0.8
spoken_word: 0.8
grammar: 0.8
context:
description: "General context for domain vocabulary and speaker names."
reporting:
enabled: true
stdout_dir: .
diagnostics:
debug: false
work_dir: /tmp/audita

View File

@@ -1,8 +0,0 @@
profiles:
local-proposal:
backend: local
model: replace-with-local-model
temperature: 0
max_tokens: 2048
top_p: 1
timeout_seconds: 120

View File

@@ -1,8 +0,0 @@
profiles:
local-validation:
backend: local
model: replace-with-local-model
temperature: 0
max_tokens: 1024
top_p: 1
timeout_seconds: 120

View File

@@ -1,8 +0,0 @@
profiles:
service-proposal-testing:
backend: service
model: replace-with-testing-model
temperature: 0
max_tokens: 2048
top_p: 1
timeout_seconds: 120

View File

@@ -1,8 +0,0 @@
profiles:
service-proposal:
backend: service
model: replace-with-provider-model
temperature: 0
max_tokens: 2048
top_p: 1
timeout_seconds: 120

View File

@@ -1,8 +0,0 @@
profiles:
service-validation-testing:
backend: service
model: replace-with-testing-model
temperature: 0
max_tokens: 1024
top_p: 1
timeout_seconds: 120

View File

@@ -1,8 +0,0 @@
profiles:
service-validation:
backend: service
model: replace-with-provider-model
temperature: 0
max_tokens: 1024
top_p: 1
timeout_seconds: 120

View File

@@ -0,0 +1,8 @@
version: 3
llm:
proposal_profile: local
pipeline:
stages:
- module: glossary
- module: grammar
glossaries: [party]

View File

@@ -0,0 +1,7 @@
party:
entries:
- id: jesters
name: Jesters
category: faction
description: A traveling group of performers.
aliases: [Jester]

View File

@@ -0,0 +1,23 @@
profiles:
- id: local
endpoint: http://localhost:11434/v1
model: example-model
api_key_required: false
collections:
audita:
prompts:
- id: modules.grammar.proposal
messages:
- role: system
content: Correct only clearly supported grammatical errors.
- role: user
content: |
{{ .ProposalContract }}
{{ .HardeningPolicy }}
{{ .TranscriptDescriptionBlock }}
Glossary: {{ .GlossaryJSON }}
Transcript: {{ .SectionJSON }}
output:
format: json
validation_mode: json_schema
schema_path: correction_set.v1.json

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Narrator",
"start": 0,
"end": 2,
"text": "The gestures gather beside the stage."
}
]

View File

@@ -0,0 +1,7 @@
version: 3
llm:
proposal_profile: service-proposal
validation_profile: service-review
pipeline:
stages:
- module: grammar

View File

@@ -0,0 +1,3 @@
llm:
proposal_profile: service-test
validation_profile: service-test

View File

@@ -0,0 +1,16 @@
backends:
- id: service
endpoint: https://example.invalid/v1
api_key_env: AUDITA_EXAMPLE_KEY
concurrency_limit: 2
queue_capacity: 8
profiles:
- id: service-proposal
backend: service
model: example-proposal-model
- id: service-review
backend: service
model: example-review-model
- id: service-test
extends: {id: service-proposal}
model: example-testing-model

View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Narrator",
"start": 0,
"end": 2,
"text": "The gestures gather beside the stage."
}
]

View File

@@ -1,6 +0,0 @@
glossary:
- name: Audita
aliases:
- audita
category: product
summary: The Audita transcript correction CLI.

22
go.mod
View File

@@ -1,21 +1,29 @@
module gitea.maximumdirect.net/eric/audita
module gitea.maximumdirect.net/eric/audita/v3
go 1.25.5
go 1.27.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
gitea.maximumdirect.net/eric/ReleaseKit v0.1.0
gitea.maximumdirect.net/eric/promptkit v0.12.0
go.maximumdirect.net/ckconfig v0.7.0
go.maximumdirect.net/ckprompt v0.7.0
go.maximumdirect.net/commandkit v0.6.0
go.maximumdirect.net/configkit v1.2.0
go.maximumdirect.net/promptkit v0.21.0
go.maximumdirect.net/releasekit v0.9.0
gopkg.in/yaml.v3 v3.0.1
)
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
github.com/spf13/cobra v1.10.1 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
tool go.maximumdirect.net/releasekit/cmd/releasekit

34
go.sum
View File

@@ -1,16 +1,9 @@
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/ReleaseKit v0.1.0 h1:Ejk85+9Of+A8M7Mod91iB3KxrrQYgnJcz9hPA6fL1a4=
gitea.maximumdirect.net/eric/ReleaseKit v0.1.0/go.mod h1:KmH607Vc/RK/zKrCB96cWh0QZYvkbxBFvtqV+BkKaxs=
gitea.maximumdirect.net/eric/promptkit v0.12.0 h1:oIFWEg6GPxtmX3Plqv0euP+FnK1LCk/t4xDy6oKCpP4=
gitea.maximumdirect.net/eric/promptkit v0.12.0/go.mod h1:pFOOy5v1zH17stEi9rmHYmfbuLjnfVMcUhfmk+4VtJY=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -22,10 +15,31 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
go.maximumdirect.net/ckconfig v0.7.0 h1:jRmKqgReLYIRHJl7Pr4+TlucifFirGANcsoC+5WrW7A=
go.maximumdirect.net/ckconfig v0.7.0/go.mod h1:ZBKCi0Z72+C6kQV+1HJHXFGcZTc11FYyaH1pQDSZFl8=
go.maximumdirect.net/ckprompt v0.7.0 h1:iACE8fZIaq9+/DF8zt1C1JwVPgZh12F2xJmlin2r45U=
go.maximumdirect.net/ckprompt v0.7.0/go.mod h1:IuCIwxqKVjpZ+Rcd2JO2xmJ1QVtmPki5dV5PnzbD8CY=
go.maximumdirect.net/commandkit v0.6.0 h1:4PHhMIlZuuvreIxZhUazeweg6zoYixEvhM9LJnkbGNA=
go.maximumdirect.net/commandkit v0.6.0/go.mod h1:ZS98DewKNVBygKVDoh5MFD09CnPlP37dS9pVsID5Eoc=
go.maximumdirect.net/configkit v1.2.0 h1:c4tDJ6Xeh/Jg90UZplLxsnf500lvrZBVj0E3i1ysir8=
go.maximumdirect.net/configkit v1.2.0/go.mod h1:s1zLJ9GGB/1Q4a+aOHMLISonJxWFgNWpA6SpPlOafeA=
go.maximumdirect.net/promptkit v0.21.0 h1:Ws4y28lnW+lIXAq4tGhXVFnQV/6AqkDChhg8knIZFrU=
go.maximumdirect.net/promptkit v0.21.0/go.mod h1:pi9acdCi80HWyzY+XYwRvMEoCiTcDWKyeBh3mzHdIFo=
go.maximumdirect.net/releasekit v0.9.0 h1:sK+Aoj6gS6ATU/1reocjwBekp4YETYENwkcX5ti6mrE=
go.maximumdirect.net/releasekit v0.9.0/go.mod h1:QE4BLrAhVJTzi+/YTcVl21K/qZp9ojRLg5kuoy3Zb44=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

110
internal/cli/app_wiring.go Normal file
View File

@@ -0,0 +1,110 @@
package cli
import (
"context"
"fmt"
"net/http"
"reflect"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
"go.maximumdirect.net/releasekit/buildinfo"
)
type invocationState struct {
engines []engineEntry
client *http.Client
}
type engineEntry struct {
selection ckconfig.Selection
engine *pk.Engine
}
func loadApplication(ctx context.Context, selection ckconfig.Selection, _ struct{}, _ *invocationState) (*ck.Result[config.Config], error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var source ck.Source
if selection.Config.Set {
if strings.TrimSpace(selection.Config.Value) == "" {
return nil, fmt.Errorf("--config requires a nonempty path")
}
source = ck.File(selection.Config.Value)
} else {
source = ck.SearchConfig("audita", ck.Searches(ck.WorkingDirectory(), ck.UserConfigDirectory(), ck.UnixSystemDirectories()))
}
return config.Load(ck.Input{Source: source}, selection.Overlay.Value, selection.Overrides...)
}
func loadPromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
loaded, err := loadApplication(ctx, selection, input, state)
if err != nil {
return nil, err
}
var client *http.Client
if state != nil {
client = state.client
}
return llm.NewEngine(ctx, loaded, client)
}
func capturePromptEngine(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
if state != nil {
for _, entry := range state.engines {
if reflect.DeepEqual(entry.selection, selection) {
return entry.engine, nil
}
}
}
engine, err := loadPromptEngine(ctx, selection, input, state)
if err != nil {
return nil, err
}
if state != nil {
state.engines = append(state.engines, engineEntry{selection: selection, engine: engine})
}
return engine, nil
}
func buildApplication(extra ...commandkit.Declaration) (*commandkit.App, error) {
return buildApplicationWithClient(nil, extra...)
}
func buildApplicationWithClient(client *http.Client, extra ...commandkit.Declaration) (*commandkit.App, error) {
appScope := ckconfig.FromConfigKit("app", "Audita application settings.", loadApplication)
appScope.Features.Overrides = true
promptScope := ckprompt.ConfigScope(loadPromptEngine)
configCommand := ckconfig.Command(ckconfig.Suite[struct{}, *invocationState]{NewState: func() *invocationState { return &invocationState{client: client} }, Scopes: []ckconfig.Scope[struct{}, *invocationState]{appScope, promptScope}})
definitions := ckprompt.WithNamespaces(ckprompt.FromEngine(capturePromptEngine), pk.Namespaces{})
definitions.NewState = func() *invocationState { return &invocationState{client: client} }
definitions = ckprompt.WithRequestValidation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.ValidationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
})
definitions = ckprompt.WithRequestPreparation(definitions, capturePromptEngine, func(_ context.Context, seed ckprompt.PreparationRequest, _ struct{}, _ *invocationState) (pk.RunRequest, error) {
return applicationPromptRequest(seed.Request, seed.Namespaces), nil
})
definitions = ckprompt.WithSmokeTesting(definitions, capturePromptEngine, nil)
declarations := []commandkit.Declaration{configCommand, commandkit.Completion("audita"), processCommand()}
declarations = append(declarations, ckprompt.Declarations(definitions)...)
declarations = append(declarations, extra...)
return commandkit.New(commandkit.Identity{Name: "audita", Version: buildinfo.Version()}, declarations...)
}
// applicationPromptRequest supplies instructions only for Audita's maintained prompts.
func applicationPromptRequest(request pk.RunRequest, namespaces *pk.Namespaces) pk.RunRequest {
id := request.PromptID
if !strings.Contains(id, "/") && namespaces != nil && namespaces.Prompt != "" {
id = namespaces.Prompt + "/" + id
}
if _, ok := prompts.LookupMetadata(id); ok {
request.Vars = prompts.WithSharedVariables(request.Vars)
}
return request
}

View File

@@ -0,0 +1,232 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
commandkit "go.maximumdirect.net/commandkit"
pk "go.maximumdirect.net/promptkit"
)
func TestNativeConfigurationAndResourceCommands(t *testing.T) {
dir := t.TempDir()
file := filepath.Join(dir, "config.yml")
if err := os.WriteFile(file, []byte("version: 3\ncontext:\n description: example\n"), 0644); err != nil {
t.Fatal(err)
}
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
args []string
contains string
}{
{[]string{"--help"}, "audita"},
{[]string{"version"}, "audita"},
{[]string{"process", "--help"}, "transcript"},
{[]string{"config", "show", "--scope", "app", "--config", file}, "example"},
{[]string{"promptkit", "list", "prompts", "--config", file}, "audita/modules.grammar.proposal"},
} {
var out, stderr bytes.Buffer
result := app.Execute(context.Background(), tc.args, commandkit.Streams{Out: &out, Err: &stderr})
if result.Status != 0 || !strings.Contains(out.String(), tc.contains) {
t.Errorf("%v: status=%d out=%q err=%q", tc.args, result.Status, out.String(), stderr.String())
}
}
}
func TestNativeConfigurationSelectionsAndComparison(t *testing.T) {
dir := t.TempDir()
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
for name, data := range map[string]string{first: "context:\n description: before\n", second: "context:\n description: after\n"} {
if err := os.WriteFile(name, []byte(data), 0644); err != nil {
t.Fatal(err)
}
}
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
var out, stderr bytes.Buffer
outcome := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--scope=app", "--config", first, "--against-config", second, "--set=/context/description=\"left\"", "--against-set=/context/description=\"right\""}, commandkit.Streams{Out: &out, Err: &stderr})
if outcome.Status != 3 || !strings.Contains(out.String(), "left") || !strings.Contains(out.String(), "right") {
t.Fatalf("status=%d output=%q err=%q", outcome.Status, out.String(), stderr.String())
}
out.Reset()
stderr.Reset()
outcome = app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config", filepath.Join(dir, "missing.yml")}, commandkit.Streams{Out: &out, Err: &stderr})
if outcome.Status == 0 {
t.Fatalf("missing file accepted: %s", out.String())
}
}
func TestExplicitEmptyConfigurationPathFails(t *testing.T) {
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
var out, stderr bytes.Buffer
result := app.Execute(context.Background(), []string{"config", "show", "--scope=app", "--config="}, commandkit.Streams{Out: &out, Err: &stderr})
if result.Status == 0 {
t.Fatal("explicit empty path accepted")
}
}
func TestPromptConfigurationComparisonAcquiresEqualSelectionsIndependently(t *testing.T) {
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
resource := filepath.Join(dir, "prompts.yml")
writeProfile := func(model string) {
t.Helper()
if err := os.WriteFile(resource, []byte("profiles: [{id: local, model: "+model+"}]\n"), 0600); err != nil {
t.Fatal(err)
}
}
writeProfile("before")
scope := ckprompt.ConfigScope(func(ctx context.Context, selection ckconfig.Selection, input struct{}, state *invocationState) (*pk.Engine, error) {
engine, err := loadPromptEngine(ctx, selection, input, state)
writeProfile("after")
return engine, err
})
side := ckconfig.ComparisonSide[struct{}]{Selection: ckconfig.Selection{
Config: commandkit.Supplied[string]{Value: main, Set: true},
}}
result, err := scope.Compare(context.Background(), ckconfig.ComparisonRequest[struct{}]{Left: side, Right: side}, &invocationState{})
if err != nil {
t.Fatal(err)
}
for _, difference := range result.Value {
if strings.Contains(difference.Left.YAML, "before") && strings.Contains(difference.Right.YAML, "after") {
return
}
}
t.Fatal("comparison did not capture the model before and after the resource changed")
}
func TestRequestInspectionAndExplicitSmokeUseSameSources(t *testing.T) {
dir := t.TempDir()
file := filepath.Join(dir, "config.yml")
if err := os.WriteFile(file, []byte("version: 3\n"), 0644); err != nil {
t.Fatal(err)
}
resources := `backends:
- id: local
endpoint: https://example.invalid/v1
smoke_tests:
default_profile: fixture
authentication:
kind: none
profiles:
- id: fixture
backend: local
model: sample
collections:
audita:
prompts:
- id: modules.grammar.proposal
messages:
- role: user
content: 'Replacement {{.HardeningPolicy}} {{.SectionJSON}}'
output:
format: text
validation_mode: none
`
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0644); err != nil {
t.Fatal(err)
}
calls := 0
client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
calls++
body := `{}`
if req.Method == http.MethodPost {
requestBody, err := io.ReadAll(req.Body)
if err != nil {
t.Fatal(err)
}
content := "PROMPTKIT_OK"
if strings.Contains(string(requestBody), "response_format") {
content = `{"status":"ok"}`
}
encoded, err := json.Marshal(content)
if err != nil {
t.Fatal(err)
}
body = `{"model":"sample","choices":[{"message":{"content":` + string(encoded) + `}}]}`
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
})}
app, err := buildApplicationWithClient(client)
if err != nil {
t.Fatal(err)
}
execute := func(args ...string) (commandkit.Outcome, string, string) {
t.Helper()
var out, stderr bytes.Buffer
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
return result, out.String(), stderr.String()
}
common := []string{"--config", file, "--llm-profile", "fixture"}
args := append([]string{"promptkit", "show", "prepared", "audita/modules.grammar.proposal"}, common...)
args = append(args, "--llm-var", "SectionJSON=[]", "--llm-var", "HardeningPolicy=custom-instruction")
outcome, out, stderr := execute(args...)
if outcome.Status != 0 || !strings.Contains(out, "Replacement custom-instruction") {
t.Fatalf("preview status=%d out=%q err=%q", outcome.Status, out, stderr)
}
args = append([]string{"promptkit", "requirements", "validate", "audita/validators.editorial_review"}, common...)
args = append(args, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
outcome, out, stderr = execute(args...)
if outcome.Status != 0 {
t.Fatalf("validator status=%d out=%q err=%q", outcome.Status, out, stderr)
}
validatorPreview := append([]string{"promptkit", "show", "prepared", "audita/validators.editorial_review"}, common...)
validatorPreview = append(validatorPreview, "--llm-var", `PayloadJSON=[{"correction_index":1,"original_segment_text":"teh cat","corrected_segment_text":"the cat"}]`, "--llm-var", "TranscriptDescriptionBlock=")
outcome, out, stderr = execute(validatorPreview...)
if outcome.Status != 0 || !strings.Contains(out, "teh cat") {
t.Fatalf("validator preview status=%d out=%q err=%q", outcome.Status, out, stderr)
}
if calls != 0 {
t.Fatalf("inspection made %d provider calls", calls)
}
outcome, out, stderr = execute("promptkit", "test", "reachability", "--config", file, "--llm-backend", "local")
if outcome.Status != 0 || calls == 0 {
t.Fatalf("smoke status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
}
previous := calls
outcome, out, stderr = execute("promptkit", "test", "all", "--config", file, "--llm-backend", "local", "--llm-profile", "fixture", "--format=json")
if outcome.Status != 0 || calls <= previous || !strings.Contains(out, "structured_output") || !strings.Contains(out, "text_inference") {
t.Fatalf("aggregate status=%d calls=%d out=%q err=%q", outcome.Status, calls, out, stderr)
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
func TestRequestDefaultsRespectNamespaceAndCallerVariables(t *testing.T) {
request := pk.RunRequest{PromptID: "modules.grammar.proposal", Vars: map[string]string{"HardeningPolicy": "caller"}}
result := applicationPromptRequest(request, &pk.Namespaces{Prompt: "audita"})
if result.Vars["HardeningPolicy"] != "caller" || result.Vars["ProposalContract"] == "" {
t.Fatalf("variables=%v", result.Vars)
}
if _, changed := request.Vars["ProposalContract"]; changed {
t.Fatal("input variables mutated")
}
generic := pk.RunRequest{PromptID: "generic", Vars: map[string]string{"x": "y"}}
result = applicationPromptRequest(generic, nil)
if len(result.Vars) != 1 || result.Vars["x"] != "y" {
t.Fatalf("generic request changed: %v", result.Vars)
}
}

View File

@@ -1,233 +0,0 @@
package cli
import (
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
var systemConfigPaths = []string{"/usr/local/etc/audita/config.yml", "/etc/audita/config.yml"}
type configInput struct {
ConfigPath string
ConfigPathSet bool
Overlay string
View string
TranscriptPath string
GlossaryPath string
OutputPath string
ReportDirSet bool
Overrides []ck.Override
}
type selectedConfig struct {
Path string
Source string
Result *ck.Result[config.Config]
}
// parseConfigInput owns the common setting flags and process-only path flags.
// A transcript before flags and one after flags are both accepted.
func parseConfigInput(command string, args []string, output io.Writer) (configInput, bool, error) {
var input configInput
fs := flag.NewFlagSet(command, flag.ContinueOnError)
fs.SetOutput(output)
configPath := fs.String("config", "", "Main configuration YAML file")
overlay := fs.String("overlay", "", "Selected configuration overlay")
if command == "print-effective" {
fs.String("view", "settings", "Effective view: settings, profiles, or explain")
}
settingPaths := map[string]ck.Path{
"proposal-profile": ck.Keys("llm", "proposal_profile"),
"validation-profile": ck.Keys("llm", "validation_profile"),
"output-schema": ck.Keys("output", "schema"),
"validation-max-prompt-tokens": ck.Keys("validation", "max_prompt_tokens"),
"target-sections": ck.Keys("chunking", "target_sections"),
"max-section-tokens": ck.Keys("chunking", "max_section_tokens"),
"min-section-tokens": ck.Keys("chunking", "min_section_tokens"),
"normalize-max-segment-gap": ck.Keys("normalization", "max_segment_gap"),
"normalize-ellipsis-gap": ck.Keys("normalization", "ellipsis_gap"),
"normalize-max-segment-duration": ck.Keys("normalization", "max_segment_duration"),
"normalize-max-segment-tokens": ck.Keys("normalization", "max_segment_tokens"),
"glossary-confidence-threshold": ck.Keys("thresholds", "glossary"),
"homophones-confidence-threshold": ck.Keys("thresholds", "homophones"),
"spoken-word-confidence-threshold": ck.Keys("thresholds", "spoken_word"),
"grammar-confidence-threshold": ck.Keys("thresholds", "grammar"),
"transcript-description": ck.Keys("context", "description"),
"reporting": ck.Keys("reporting", "enabled"),
"report-dir": ck.Keys("reporting", "stdout_dir"),
"debug": ck.Keys("diagnostics", "debug"),
"work-dir": ck.Keys("diagnostics", "work_dir"),
"modules": ck.Keys("pipeline", "modules"),
}
for name := range settingPaths {
switch name {
case "reporting", "debug":
fs.Bool(name, false, "Setting override")
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
fs.Int(name, 0, "Setting override")
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
fs.Float64(name, 0, "Setting override")
default:
fs.String(name, "", "Setting override")
}
}
if command == "process" {
fs.String("glossary", "", "Glossary YAML file")
fs.String("output", "", "Corrected transcript output file")
}
parseArgs := args
front := ""
if command == "process" && len(args) > 0 && !strings.HasPrefix(args[0], "-") {
front, parseArgs = args[0], args[1:]
}
if err := fs.Parse(parseArgs); err != nil {
if errors.Is(err, flag.ErrHelp) {
return configInput{}, true, nil
}
return configInput{}, false, err
}
positions := fs.Args()
if front != "" {
positions = append([]string{front}, positions...)
}
if command == "process" {
if len(positions) != 1 {
return configInput{}, false, fmt.Errorf("expected exactly one transcript JSON path")
}
input.TranscriptPath = positions[0]
input.GlossaryPath = fs.Lookup("glossary").Value.String()
input.OutputPath = fs.Lookup("output").Value.String()
if strings.TrimSpace(input.GlossaryPath) == "" {
return configInput{}, false, fmt.Errorf("--glossary is required")
}
} else if len(positions) != 0 {
return configInput{}, false, fmt.Errorf("unexpected positional arguments")
}
input.ConfigPath, input.Overlay = *configPath, *overlay
if command == "print-effective" {
input.View = fs.Lookup("view").Value.String()
switch input.View {
case "settings", "profiles", "explain":
default:
return configInput{}, false, fmt.Errorf("unknown effective view %q", input.View)
}
}
fs.Visit(func(f *flag.Flag) {
if f.Name == "config" {
input.ConfigPathSet = true
}
if f.Name == "report-dir" {
input.ReportDirSet = true
}
path, ok := settingPaths[f.Name]
if !ok {
return
}
var value any = f.Value.String()
switch f.Name {
case "reporting", "debug":
value, _ = strconv.ParseBool(f.Value.String())
case "modules":
modules, err := config.ParseModulesCSV(f.Value.String())
if err != nil {
value = err
} else {
values := make([]any, len(modules))
for i, module := range modules {
values[i] = module
}
value = values
}
case "validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens", "normalize-max-segment-tokens":
value, _ = strconv.Atoi(f.Value.String())
case "normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold":
value, _ = strconv.ParseFloat(f.Value.String(), 64)
}
input.Overrides = append(input.Overrides, ck.Override{Path: path, Value: value, Flag: "--" + f.Name})
})
for _, override := range input.Overrides {
if err, ok := override.Value.(error); ok {
return configInput{}, false, fmt.Errorf("%s: %w", override.Flag, err)
}
}
return input, false, nil
}
func selectConfigPath(explicit string, set bool, stat func(string) (os.FileInfo, error), candidates []string) (string, string, error) {
if set {
if strings.TrimSpace(explicit) == "" {
return "", "", fmt.Errorf("--config requires a nonempty path")
}
if _, err := stat(explicit); err != nil {
return "", "", fmt.Errorf("config file %q: %w", explicit, err)
}
return explicit, "flag", nil
}
for _, path := range candidates {
if _, err := stat(path); err == nil {
return path, "default", nil
} else if !os.IsNotExist(err) {
return "", "", fmt.Errorf("config file %q: %w", path, err)
}
}
return "", "", fmt.Errorf("no configuration file found; select a config with backend and profile settings")
}
func loadConfigInput(input configInput, command string) (selectedConfig, error) {
if command == "validate" && (!input.ConfigPathSet || strings.TrimSpace(input.ConfigPath) == "") {
return selectedConfig{}, fmt.Errorf("--config is required")
}
path, source, err := selectConfigPath(input.ConfigPath, input.ConfigPathSet, os.Stat, systemConfigPaths)
if err != nil {
return selectedConfig{}, err
}
loaded, err := config.Load(ck.File(path), input.Overlay, input.Overrides...)
if err != nil {
return selectedConfig{}, err
}
return selectedConfig{Path: path, Source: source, Result: loaded}, nil
}
func configInputError(err error) string {
var loadErr *ck.LoadError
if !errors.As(err, &loadErr) {
return err.Error()
}
var messages []string
for _, diagnostic := range loadErr.Diagnostics() {
messages = append(messages, diagnostic.Message)
}
if len(messages) == 0 {
return err.Error()
}
return strings.Join(messages, "; ")
}
func selectedReportDir(cfg config.Config, outputPath string, reportDirSet bool) (string, error) {
if !cfg.Reporting.Enabled {
return "", nil
}
var reportDir string
if outputPath != "" {
if reportDirSet {
reportDir = cfg.Reporting.StdoutDir
} else {
reportDir = filepath.Dir(outputPath)
}
} else {
reportDir = cfg.Reporting.StdoutDir
}
if strings.TrimSpace(reportDir) == "" {
return "", fmt.Errorf("report directory is required when reporting is enabled for the selected destination")
}
return reportDir, nil
}

View File

@@ -1,280 +0,0 @@
package cli
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
)
func TestSharedConfigInput(t *testing.T) {
for _, command := range []string{"process", "validate", "print-effective"} {
t.Run(command, func(t *testing.T) {
args := []string{"--config", "main.yml", "--overlay", "testing", "--proposal-profile", "economy", "--modules", "", "--target-sections", "0"}
if command == "process" {
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
}
input, help, err := parseConfigInput(command, args, io.Discard)
if err != nil || help {
t.Fatalf("parse: %+v %v %v", input, help, err)
}
if !input.ConfigPathSet || input.ConfigPath != "main.yml" || input.Overlay != "testing" || len(input.Overrides) != 3 {
t.Fatalf("selection: %+v", input)
}
for _, override := range input.Overrides {
switch override.Path.String() {
case ck.Keys("pipeline", "modules").String():
if values, ok := override.Value.([]any); !ok || len(values) != 0 {
t.Fatalf("empty modules: %#v", override.Value)
}
case ck.Keys("chunking", "target_sections").String():
if override.Value != 0 {
t.Fatalf("zero target: %#v", override.Value)
}
}
}
})
}
}
func TestConfigInputRejectsRemovedAndProcessOnlyFlags(t *testing.T) {
for _, command := range []string{"process", "validate", "print-effective"} {
for _, flag := range []string{"--model", "--llm-api-key", "--max-retries", "--llm-concurrency", "--report-json", "--work-dir-retention"} {
args := []string{flag, "value"}
if command == "process" {
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
}
if _, _, err := parseConfigInput(command, args, io.Discard); err == nil {
t.Fatalf("%s accepted %s", command, flag)
}
}
}
for _, command := range []string{"validate", "print-effective"} {
if _, _, err := parseConfigInput(command, []string{"--glossary", "glossary.yml"}, io.Discard); err == nil {
t.Fatalf("%s accepted process path", command)
}
if _, _, err := parseConfigInput(command, []string{"transcript.json"}, io.Discard); err == nil {
t.Fatalf("%s accepted positional", command)
}
}
for _, command := range []string{"process", "validate", "print-effective"} {
_, help, err := parseConfigInput(command, []string{"--help"}, io.Discard)
if err != nil || !help {
t.Fatalf("%s help: %v %v", command, help, err)
}
}
}
func TestConfigSelection(t *testing.T) {
dir := t.TempDir()
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
if err := os.WriteFile(second, []byte("version: 2\n"), 0600); err != nil {
t.Fatal(err)
}
path, source, err := selectConfigPath("", false, os.Stat, []string{first, second})
if err != nil || path != second || source != "default" {
t.Fatalf("discovery: %q %q %v", path, source, err)
}
for _, tc := range []struct {
explicit string
set bool
candidates []string
}{
{"", true, []string{second}}, {first, true, []string{second}}, {"", false, []string{first}},
} {
if _, _, err := selectConfigPath(tc.explicit, tc.set, os.Stat, tc.candidates); err == nil {
t.Fatalf("accepted unavailable selection: %+v", tc)
}
}
statDenied := func(string) (os.FileInfo, error) { return nil, os.ErrPermission }
if _, _, err := selectConfigPath("", false, statDenied, []string{first}); !errors.Is(err, os.ErrPermission) {
t.Fatalf("inaccessible candidate: %v", err)
}
path, source, err = selectConfigPath(second, true, os.Stat, []string{first})
if err != nil || path != second || source != "flag" {
t.Fatalf("explicit path: %q %q %v", path, source, err)
}
}
func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) {
t.Setenv("AUDITA_CONFIG", filepath.Join(t.TempDir(), "legacy.yml"))
_, _, err := selectConfigPath("", false, os.Stat, []string{filepath.Join(t.TempDir(), "missing.yml")})
if err == nil || !strings.Contains(err.Error(), "no configuration file") {
t.Fatalf("legacy environment changed discovery: %v", err)
}
}
func TestEffectiveViewParsing(t *testing.T) {
for _, view := range []string{"settings", "profiles", "explain"} {
input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard)
if err != nil || input.View != view {
t.Fatalf("%q: %+v %v", view, input, err)
}
}
if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil {
t.Fatal("unknown view accepted")
}
if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil {
t.Fatal("validate accepted inspection-only flag")
}
}
func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\nchunking:\n max_section_tokens: 900\n"
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\n"), 0600); err != nil {
t.Fatal(err)
}
for _, command := range []string{"process", "validate", "print-effective"} {
args := []string{"--config", main, "--overlay", "testing", "--min-section-tokens", "500"}
if command == "process" {
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
}
input, _, err := parseConfigInput(command, args, io.Discard)
if err != nil {
t.Fatal(err)
}
selected, err := loadConfigInput(input, command)
if err != nil {
t.Fatal(err)
}
c := selected.Result.Config
if c.LLM.ProposalProfile != "testing" || c.LLM.ValidationProfile != "testing" || c.Chunking.MinSectionTokens != 500 || c.Chunking.MaxSectionTokens != 900 {
t.Fatalf("%s resolved %+v", command, c)
}
}
input, _, err := parseConfigInput("validate", []string{"--overlay", "testing"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if _, err := loadConfigInput(input, "validate"); err == nil {
t.Fatal("validate accepted discovery")
}
input, _, err = parseConfigInput("validate", []string{"--config", main}, io.Discard)
if err != nil {
t.Fatal(err)
}
if _, err := loadConfigInput(input, "validate"); err == nil {
t.Fatal("unmodified invalid lower-precedence section relation accepted")
}
}
func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\n"
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("reporting:\n enabled: true\n stdout_dir: reports\ndiagnostics:\n debug: true\n"), 0600); err != nil {
t.Fatal(err)
}
defaultInput, _, err := parseConfigInput("validate", []string{"--config", main}, io.Discard)
if err != nil {
t.Fatal(err)
}
defaults, err := loadConfigInput(defaultInput, "validate")
if err != nil {
t.Fatal(err)
}
if !defaults.Result.Config.Reporting.Enabled || defaults.Result.Config.Reporting.StdoutDir != "." || defaults.Result.Config.Diagnostics.Debug {
t.Fatalf("unexpected defaults: %+v", defaults.Result.Config)
}
for _, command := range []string{"process", "validate", "print-effective"} {
args := []string{"--config", main, "--overlay", "testing", "--reporting=false", "--debug=false"}
if command == "process" {
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
}
input, _, err := parseConfigInput(command, args, io.Discard)
if err != nil {
t.Fatal(err)
}
if len(input.Overrides) != 2 || input.Overrides[0].Value != false || input.Overrides[1].Value != false {
t.Fatalf("boolean overrides must remain typed: %+v", input.Overrides)
}
selected, err := loadConfigInput(input, command)
if err != nil {
t.Fatal(err)
}
if selected.Result.Config.Reporting.Enabled || selected.Result.Config.Diagnostics.Debug || selected.Result.Config.Reporting.StdoutDir != "reports" {
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
}
}
processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard)
if err != nil {
t.Fatal(err)
}
processConfig, err := loadConfigInput(processInput, "process")
if err != nil {
t.Fatal(err)
}
if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" {
t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err)
}
}
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" {
t.Fatalf("stdout destination: %q %v", got, err)
}
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
t.Fatalf("file destination ignored stdout setting: %q %v", got, err)
}
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "reports" {
t.Fatalf("explicit file report directory: %q %v", got, err)
}
cfg.Reporting.StdoutDir = ""
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), false); err != nil || got != "out" {
t.Fatalf("inactive blank configured report directory rejected for file output: %q %v", got, err)
}
if _, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err == nil {
t.Fatal("active explicit blank report directory accepted for file output")
}
if _, err := selectedReportDir(cfg, "", false); err == nil {
t.Fatal("enabled stdout reporting accepted blank destination")
}
cfg.Reporting.Enabled = false
if got, err := selectedReportDir(cfg, filepath.Join("out", "corrected.json"), true); err != nil || got != "" {
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
}
}
func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
input, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--reporting", "--debug", "--report-dir", "reports"}, io.Discard)
if err != nil {
t.Fatal(err)
}
values := map[string]any{}
for _, override := range input.Overrides {
values[override.Path.String()] = override.Value
}
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
t.Fatalf("incorrect visited overrides: %+v", values)
}
if !input.ReportDirSet {
t.Fatal("explicit --report-dir presence was not recorded")
}
withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if withoutReportDir.ReportDirSet {
t.Fatal("absent --report-dir was recorded as explicit")
}
}

View File

@@ -0,0 +1,156 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
ck "go.maximumdirect.net/configkit"
)
func TestMaintainedProjectsWithControlledProvider(t *testing.T) {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("cannot locate examples")
}
root := filepath.Join(filepath.Dir(file), "..", "..", "examples")
t.Setenv("AUDITA_EXAMPLE_KEY", "synthetic-example-key")
for _, tc := range []struct{ dir, main, overlay string }{
{"local", "config.yml", ""}, {"service", "config.yml", "testing"},
{"prompt-edit", "config.yml", ""}, {"campaigns", "first.yml", ""}, {"campaigns", "second.yml", ""},
} {
t.Run(tc.dir+"/"+tc.main, func(t *testing.T) {
main := filepath.Join(root, tc.dir, tc.main)
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, tc.overlay)
if err != nil {
t.Fatal(err)
}
loaded.Config.Reporting.Enabled = false
var calls atomic.Int32
var mu sync.Mutex
var models, prompts []string
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
calls.Add(1)
var request struct {
Model string `json:"model"`
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
}
var content strings.Builder
for _, message := range request.Messages {
content.WriteString(message.Content)
}
mu.Lock()
models = append(models, request.Model)
prompts = append(prompts, content.String())
mu.Unlock()
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":"{\"corrections\":[]}"}}]}`)), Request: r}, nil
})
engine, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
if tc.dir == "prompt-edit" {
prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
if err != nil || !strings.Contains(prompt.Messages[0].Content, "clearly supported grammatical errors") {
t.Fatalf("operator prompt: %+v %v", prompt, err)
}
}
raw, err := os.ReadFile(filepath.Join(root, tc.dir, "transcript.json"))
if err != nil {
t.Fatal(err)
}
source, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
t.Fatal(err)
}
inputs := &processInputs{Configuration: loaded, Runtime: engine, Source: source}
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(root, tc.dir, "transcript.json")}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
if _, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {}); err != nil {
t.Fatal(err)
}
if int(calls.Load()) != len(loaded.Config.Pipeline.Stages) {
t.Fatalf("provider calls=%d", calls.Load())
}
if tc.dir == "service" && (len(models) != 1 || models[0] != "example-testing-model") {
t.Fatalf("overlay models=%v", models)
}
if tc.dir == "campaigns" && tc.main == "first.yml" {
if len(prompts) != 3 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) || !strings.Contains(prompts[1], `"name": "Shield"`) || strings.Contains(prompts[1], `"name": "Lyra"`) {
t.Fatalf("campaign glossary selections: %v", prompts)
}
}
if tc.dir == "campaigns" && tc.main == "second.yml" {
if len(prompts) != 2 || !strings.Contains(prompts[0], `"name": "Lyra"`) || strings.Contains(prompts[0], `"name": "Shield"`) {
t.Fatalf("second campaign glossary selection: %v", prompts)
}
}
})
}
}
func TestExamplePromptEditIsCapturedOnNextLoad(t *testing.T) {
_, file, _, ok := runtime.Caller(0)
if !ok {
t.Fatal("cannot locate examples")
}
source := filepath.Join(filepath.Dir(file), "..", "..", "examples", "prompt-edit")
dir := t.TempDir()
for _, name := range []string{"config.yml", "glossary.yml", "prompts.yml"} {
raw, err := os.ReadFile(filepath.Join(source, name))
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, name), raw, 0600); err != nil {
t.Fatal(err)
}
}
inspect := func() string {
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
engine, err := llm.NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
prompt, err := engine.Engine.InspectPrompt(context.Background(), "audita/modules.grammar.proposal")
if err != nil {
t.Fatal(err)
}
return prompt.Messages[0].Content
}
before := inspect()
filePath := filepath.Join(dir, "prompts.yml")
raw, err := os.ReadFile(filePath)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filePath, []byte(strings.Replace(string(raw), "clearly supported grammatical errors", "clearly supported punctuation errors", 1)), 0600); err != nil {
t.Fatal(err)
}
after := inspect()
if !strings.Contains(before, "grammatical errors") || !strings.Contains(after, "punctuation errors") {
t.Fatalf("before=%q after=%q", before, after)
}
}

View File

@@ -1,442 +1,92 @@
package cli
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"strings"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/internal/core/io"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
"gitea.maximumdirect.net/eric/audita/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/processreport"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"go.maximumdirect.net/ckconfig"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
func runProcess(ctx context.Context, args []string, stdout, stderr io.Writer) int {
input, help, err := parseConfigInput("process", args, stderr)
// processResult carries the runner's transcript and ordinary report facts.
type processResult struct {
Transcript *schema.Transcript
Snapshot reporting.Snapshot
}
func runProcess(inv commandkit.Invocation, input processInput) commandkit.Outcome {
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
return 2
}
if help {
writeProcessUsage(stdout)
return 0
}
selected, err := loadConfigInput(input, "process")
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", configInputError(err))
return 2
}
cfg := selected.Result.Config
reportDir, err := selectedReportDir(cfg, input.OutputPath, input.ReportDirSet)
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 2
}
if cfg.Diagnostics.Debug && strings.TrimSpace(cfg.Diagnostics.WorkDir) == "" {
fmt.Fprintln(stderr, "audita process: debug work directory is required")
return 2
}
if err := ctx.Err(); err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", err)
return 1
}
runtime, err := llm.NewRuntime(ctx, cfg, selected.Path, nil)
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
if ctx.Err() != nil || llm.IsAssetError(err) {
return 1
if inv.Context.Err() != nil || llm.IsAssetError(err) {
return commandkit.Failure("Could not prepare processing.").WithCause(err)
}
return 2
}
specs, err := contracts.ResolveModuleRunSpecs(cfg.Pipeline.Modules)
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 2
}
runID, err := generateRunID()
if err != nil {
fmt.Fprintf(stderr, "audita process: run ID: %s\n", llm.RedactError(err, cfg))
return 1
}
started := time.Now().UTC()
state := &processState{cfg: cfg, runtime: runtime, specs: specs, runID: runID, started: started,
transcriptPath: input.TranscriptPath, glossaryPath: input.GlossaryPath, outputPath: input.OutputPath,
selected: selected, overlay: input.Overlay, stderr: stderr, stdout: stdout, ctx: ctx}
protected := []string{input.GlossaryPath}
for _, source := range selected.Result.Sources() {
if source.Path != "" {
protected = append(protected, source.Path)
var loadErr *ck.LoadError
if errors.As(err, &loadErr) {
_, explanation := ckconfig.AdaptDiagnostics("Application configuration:\n", loadErr.Sources(), loadErr.Diagnostics())
return commandkit.Usage(loadErr.Error(), explanation.Details...).WithCause(err)
}
return commandkit.Usage(err.Error()).WithCause(err)
}
protected = append(protected, runtime.SourcePaths...)
session, err := diagnostics.PrepareOutput(diagnostics.OutputSelection{InputTranscriptPath: input.TranscriptPath, TranscriptPath: input.OutputPath,
ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: protected})
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 2
return commandkit.Failure(loaded.Runtime.RedactError(inv.Context, err)).WithCause(err)
}
state.session = session
initial := state.snapshot(reporting.StatusInProgress, nil, nil)
initialBundle, err := state.bundle(initial)
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 1
defer prepared.Prepared.Output.Close()
var warningsMu sync.Mutex
warn := func(err error) {
if err == nil {
return
}
warningsMu.Lock()
prepared.Prepared.Warnings = append(prepared.Prepared.Warnings, err)
warningsMu.Unlock()
}
initFailure := []processreport.RunFailure{{Phase: "initialization", Reason: outcomes.ReasonInternalFailure}}
failedInit, err := state.bundle(state.snapshot(reporting.StatusFailed, nil, initFailure))
if err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 1
debug := prepared.Prepared.Debug
if debug != nil {
warn(debug.WriteSourceTranscript(loaded.Source, loaded.RawTranscript))
}
if err := session.Initialize(initialBundle, failedInit); err != nil {
fmt.Fprintf(stderr, "audita process: %s\n", llm.RedactError(err, cfg))
return 1
}
state.startDebug()
return state.execute()
}
type processState struct {
cfg config.Config
runtime *llm.Runtime
specs []contracts.ModuleRunSpec
runID string
started time.Time
transcriptPath string
glossaryPath string
outputPath string
selected selectedConfig
overlay string
ctx context.Context
stdout io.Writer
stderr io.Writer
session *diagnostics.OutputSession
debugDir *diagnostics.RunDirectory
debugWarnings []string
debugMu sync.Mutex
runOutput *runner.RunOutput
}
func generateRunID() (string, error) {
var random [12]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
return "run-" + hex.EncodeToString(random[:]), nil
}
func (s *processState) snapshot(status string, completed *time.Time, failures []processreport.RunFailure) reporting.Snapshot {
return processreport.Build(processreport.BuildInput{RunID: s.runID, Status: status, StartedAt: s.started,
CompletedAt: completed, TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
OutputPath: s.outputPath, ModuleSpecs: s.specs, RunOutput: s.runOutput, RunFailures: failures})
}
func (s *processState) bundle(snapshot reporting.Snapshot) (diagnostics.ReportBundle, error) {
if !s.cfg.Reporting.Enabled {
return diagnostics.ReportBundle{}, nil
}
secrets := llm.ConfiguredSecrets(&s.cfg)
errorsYAML, err := reporting.YAML(snapshot.ErrorsDocument(), secrets...)
if err != nil {
return diagnostics.ReportBundle{}, err
}
recoveriesYAML, err := reporting.YAML(snapshot.RecoveriesDocument(), secrets...)
if err != nil {
return diagnostics.ReportBundle{}, err
}
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML,
Markdown: reporting.Markdown(snapshot, secrets...)}, nil
}
func (s *processState) debugWrite(label string, write func(*diagnostics.RunDirectory) error) {
if s.debugDir == nil {
return
}
if err := write(s.debugDir); err != nil {
s.reportDiagnosticError(fmt.Errorf("%s: %w", label, err))
}
}
func (s *processState) startDebug() {
if !s.cfg.Diagnostics.Debug {
return
}
dir, err := diagnostics.NewRunDirectory(s.cfg.Diagnostics.WorkDir, s.runID, llm.ConfiguredSecrets(&s.cfg)...)
if err != nil {
s.reportDiagnosticError(fmt.Errorf("directory: %w", err))
return
}
s.debugDir = dir
s.debugWrite("invocation", func(d *diagnostics.RunDirectory) error {
return d.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process", TranscriptPath: s.transcriptPath, GlossaryPath: s.glossaryPath,
OutputPath: s.outputPath, ConfigPath: s.selected.Path, ConfigSource: s.selected.Source,
Overlay: s.overlay, ConfigVersion: &s.cfg.Version, TranscriptDescription: s.cfg.Context.Description,
Modules: append([]string(nil), s.cfg.Pipeline.Modules...), RunID: s.runID, StartedAt: s.started,
})
cfg := &loaded.Configuration.Config
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: cfg.Normalization.MaxSegmentGap, EllipsisGap: cfg.Normalization.EllipsisGap,
MaxSegmentDuration: cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: cfg.Normalization.MaxSegmentTokens,
})
var effective bytes.Buffer
if err := s.selected.Result.WriteEffective(&effective); err == nil {
s.debugWrite("effective settings", func(d *diagnostics.RunDirectory) error { return d.WriteEffectiveConfig(effective.Bytes()) })
} else {
s.reportDiagnosticError(fmt.Errorf("effective settings: %w", err))
normalized, summary := normalizer.Normalize(sourceToCanonicalTranscript(loaded.Source))
if debug != nil {
warn(debug.WriteNormalizedTranscript(normalized))
warn(debug.WriteNormalizationSummary(summary))
}
if origins, err := llm.ExplanationYAML(s.selected.Result); err == nil {
s.debugWrite("configuration origins", func(d *diagnostics.RunDirectory) error {
return d.WriteYAMLArtifact(diagnostics.ArtifactConfigOrigins, origins)
})
} else {
s.reportDiagnosticError(fmt.Errorf("configuration origins: %w", err))
if err := inv.Context.Err(); err != nil {
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, nil, err)
}
if profiles, err := s.runtime.SelectedProfilesYAML(); err == nil {
s.debugWrite("selected profiles", func(d *diagnostics.RunDirectory) error {
return d.WriteYAMLArtifact(diagnostics.ArtifactSelectedProfiles, profiles)
})
} else {
s.reportDiagnosticError(fmt.Errorf("selected profiles: %w", err))
result, runErr := executePipeline(inv.Context, loaded, prepared, normalized, warn)
if runErr != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, runErr)
}
encoder, err := outputschema.Resolve(cfg.Output.Schema)
if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
}
if result.Transcript == nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, fmt.Errorf("processing returned no transcript"))
}
payload, err := encoder.Encoder(result.Transcript)
if err != nil {
return finishProcess(inv, input, loaded, prepared, result.Snapshot, nil, err)
}
return finishProcess(inv, input, loaded, prepared, result.Snapshot, payload, nil)
}
func (s *processState) debugPath() string {
if s.debugDir == nil {
// DebugPath returns the retained artifact directory, if its preparation succeeded.
func (p *preparedProcess) DebugPath() string {
if p == nil || p.Prepared == nil || p.Prepared.Debug == nil {
return ""
}
return s.debugDir.Path()
}
func (s *processState) execute() int {
transcriptBytes, err := coreio.ReadRequiredFile(s.transcriptPath, "transcript")
if err != nil {
return s.fail("input", outcomes.ReasonInternalFailure, err)
}
glossaryBytes, err := coreio.ReadRequiredFile(s.glossaryPath, "glossary")
if err != nil {
return s.fail("input", outcomes.ReasonInternalFailure, err)
}
source, err := schema.ParseSourceTranscriptJSON(transcriptBytes)
if err != nil {
return s.fail("input", outcomes.ReasonInputInvalid, err)
}
glossary, err := schema.ParseGlossaryYAML(glossaryBytes)
if err != nil {
return s.fail("input", outcomes.ReasonInputInvalid, err)
}
s.debugWrite("source transcript", func(d *diagnostics.RunDirectory) error { return d.WriteSourceTranscript(source, transcriptBytes) })
normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{
MaxSegmentGap: s.cfg.Normalization.MaxSegmentGap, EllipsisGap: s.cfg.Normalization.EllipsisGap,
MaxSegmentDuration: s.cfg.Normalization.MaxSegmentDuration, MaxSegmentTokens: s.cfg.Normalization.MaxSegmentTokens,
})
normalized, normSummary := normalizer.Normalize(sourceToCanonicalTranscript(source))
s.debugWrite("normalized transcript", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizedTranscript(normalized) })
s.debugWrite("normalization summary", func(d *diagnostics.RunDirectory) error { return d.WriteNormalizationSummary(normSummary) })
if err := s.ctx.Err(); err != nil {
return s.fail("processing", outcomes.ReasonCanceled, err)
}
observations := llm.NewObservationCollector()
proposalClient, err := s.runtime.NewRoleClient("proposal", observations.Hooks())
if err != nil {
return s.fail("processing", outcomes.ReasonSetupFailure, err)
}
validationClient, err := s.runtime.NewRoleClient("validation", observations.Hooks())
if err != nil {
return s.fail("processing", outcomes.ReasonSetupFailure, err)
}
factory := modules.NewFactory(modules.Dependencies{Config: &s.cfg, Glossary: glossary, DiagnosticsDir: s.debugPath()})
result, runErr := runner.New(factory).Run(s.ctx, runner.RunInput{
Config: &s.cfg, Transcript: normalized, Glossary: glossary, ModuleSpecs: s.specs,
ProposalLLMClient: proposalClient, ProposalDiagnosticsDir: s.debugPath(),
ValidationLLMClient: validationClient, ValidationDiagnosticsDir: s.debugPath(),
CaptureCandidates: s.debugDir != nil,
ReportDiagnosticError: s.reportDiagnosticError,
})
result.Observations = observations.Snapshot()
result.ObservationsAvailable = true
result.LLMSummary = llm.Summarize(result.Observations)
s.runOutput = &result
s.writeExecutionEvidence()
if len(result.Sections) > 0 || runErr == nil {
s.writeChunkSummary(result.Sections)
}
if runErr != nil {
return s.fail("processing", fatalReason(s.ctx, runErr), runErr)
}
encoder, err := outputschema.Resolve(s.cfg.Output.Schema)
if err != nil {
return s.fail("encoding", outcomes.ReasonSetupFailure, err)
}
outputBytes, err := encoder.Encoder(result.FinalTranscript)
if err != nil {
return s.fail("encoding", outcomes.ReasonInternalFailure, err)
}
return s.finish(outputBytes)
}
func (s *processState) writeChunkSummary(inventory []contracts.SectionMetadata) {
if s.debugDir == nil {
return
}
sections := make([]chunking.Section, len(inventory))
for i, metadata := range inventory {
sections[i] = chunking.Section{Index: metadata.Index, StartSegmentID: metadata.StartSegmentID,
EndSegmentID: metadata.EndSegmentID, EstimatedTokens: metadata.EstimatedTokens,
Segments: make([]schema.Segment, metadata.SegmentCount)}
}
config := chunking.ChunkingConfig{TargetSections: s.cfg.Chunking.TargetSections,
MaxSectionTokens: s.cfg.Chunking.MaxSectionTokens, MinSectionTokens: s.cfg.Chunking.MinSectionTokens}
summary := chunking.ComputeDetailedSummary(sections, config)
s.debugWrite("chunking summary", func(d *diagnostics.RunDirectory) error { return d.WriteChunkingSummary(&summary) })
}
func (s *processState) writeExecutionEvidence() {
if s.debugDir == nil || s.runOutput == nil {
return
}
output := s.runOutput
s.debugWrite("model execution", func(d *diagnostics.RunDirectory) error {
return d.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations))
})
s.debugWrite("utilization", func(d *diagnostics.RunDirectory) error {
return d.WriteJSONArtifact(diagnostics.ArtifactUtilizationSummary, processreport.BuildUtilization(output, s.cfg.Backends))
})
s.debugWrite("correction ledger", func(d *diagnostics.RunDirectory) error {
return d.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{
RunDirectoryPath: d.Path(), RunOutput: output}))
})
}
func fatalReason(ctx context.Context, err error) outcomes.ReasonCode {
if ctx.Err() != nil || errors.Is(err, context.Canceled) {
return outcomes.ReasonCanceled
}
var failure *structuredoutput.Failure
if errors.As(err, &failure) && failure.Reason != "" {
return failure.Reason
}
return outcomes.ReasonInternalFailure
}
func (s *processState) fail(phase string, reason outcomes.ReasonCode, cause error) int {
s.debugWrite("error log", func(d *diagnostics.RunDirectory) error {
return d.WriteErrorLog(phase + ": " + llm.RedactError(cause, s.cfg))
})
failures := []processreport.RunFailure{{Phase: phase, Reason: reason}}
completed := time.Now().UTC()
base, err := s.bundle(s.snapshot(reporting.StatusFailed, &completed, failures))
if err != nil {
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(err, s.cfg))
}
writeFailures := s.session.Finalize(nil, base, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
all := append(append([]processreport.RunFailure(nil), failures...), mapWriteFailures(extra)...)
bundle, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, all))
if renderErr != nil {
return base
}
return bundle
})
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(cause, s.cfg))
s.printFailures(writeFailures)
s.printDebugWarnings()
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
}
return 1
}
func (s *processState) finish(outputBytes []byte) int {
completed := time.Now().UTC()
snapshot := s.snapshot(reporting.StatusCompleted, &completed, nil)
complete, err := s.bundle(snapshot)
if err != nil {
return s.fail("report_write", outcomes.ReasonInternalFailure, err)
}
if s.session.TranscriptPath() == "" {
if _, err := s.stdout.Write(outputBytes); err != nil {
return s.fail("transcript_write", outcomes.ReasonInternalFailure, err)
}
outputBytes = nil
}
writeFailures := s.session.Finalize(outputBytes, complete, func(extra []diagnostics.FileFailure) diagnostics.ReportBundle {
failed, renderErr := s.bundle(s.snapshot(reporting.StatusFailed, &completed, mapWriteFailures(extra)))
if renderErr != nil {
return diagnostics.ReportBundle{}
}
return failed
})
if len(writeFailures) > 0 {
s.printFailures(writeFailures)
s.printDebugWarnings()
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "audita process: diagnostics: %s\n", s.safePath(s.debugDir.Path()))
}
return 1
}
fmt.Fprintln(s.stderr, reporting.CompletionLine(snapshot))
for _, path := range s.session.ReportPaths() {
fmt.Fprintf(s.stderr, "Report: %s\n", s.safePath(path))
}
if s.debugDir != nil {
fmt.Fprintf(s.stderr, "Debug: %s\n", s.safePath(s.debugDir.Path()))
}
s.printDebugWarnings()
return 0
}
func mapWriteFailures(failures []diagnostics.FileFailure) []processreport.RunFailure {
out := make([]processreport.RunFailure, len(failures))
for i, failure := range failures {
phase := "report_write"
if failure.Kind == "transcript write" {
phase = "transcript_write"
}
out[i] = processreport.RunFailure{Phase: phase, Reason: outcomes.ReasonInternalFailure, OutputPath: failure.Path}
}
return out
}
func (s *processState) printFailures(failures []diagnostics.FileFailure) {
for _, failure := range failures {
fmt.Fprintf(s.stderr, "audita process: %s\n", llm.RedactError(failure, s.cfg))
}
}
func (s *processState) printDebugWarnings() {
s.debugMu.Lock()
defer s.debugMu.Unlock()
for _, warning := range s.debugWarnings {
fmt.Fprintf(s.stderr, "audita process: %s\n", warning)
}
}
// safePath sanitizes presentation only; filesystem destinations remain literal.
func (s *processState) safePath(path string) string {
return redaction.New(llm.ConfiguredSecrets(&s.cfg)).String(path)
}
func (s *processState) reportDiagnosticError(err error) {
if err == nil {
return
}
message := "debug " + llm.RedactError(err, s.cfg)
s.debugMu.Lock()
defer s.debugMu.Unlock()
s.debugWarnings = append(s.debugWarnings, message)
return p.Prepared.Debug.Path()
}

View File

@@ -0,0 +1,120 @@
package cli
import (
"fmt"
"strconv"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
type processInput struct {
Config commandkit.Supplied[string]
Overlay commandkit.Supplied[string]
Transcript string
Output commandkit.Supplied[string]
Debug commandkit.Supplied[bool]
Overrides []ck.Override
}
func processCommand() commandkit.Command {
stringValue := func(s string) (string, error) { return s, nil }
flags := []commandkit.Flag[processInput]{
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Application configuration file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Config = v }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Configuration overlay."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Overlay = v }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "output", Usage: "Transcript output file."}, (*string)(nil), stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Output = v }),
commandkit.BoolFlag(commandkit.FlagMeta{Name: "debug", Usage: "Capture diagnostic artifacts."}, false, func(in *processInput, v commandkit.Supplied[bool]) { in.Debug = v }),
}
for _, item := range []struct {
name string
path ck.Path
}{
{"proposal-profile", ck.Keys("llm", "proposal_profile")}, {"validation-profile", ck.Keys("llm", "validation_profile")},
{"output-schema", ck.Keys("output", "schema")}, {"transcript-description", ck.Keys("context", "description")},
{"report-dir", ck.Keys("reporting", "stdout_dir")}, {"work-dir", ck.Keys("diagnostics", "work_dir")},
} {
name, path := item.name, item.path
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*string)(nil), stringValue,
func(in *processInput, v commandkit.Supplied[string]) {
if v.Set {
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
}
}))
}
for _, item := range []struct {
name string
path ck.Path
}{
{"validation-max-prompt-tokens", ck.Keys("validation", "max_prompt_tokens")}, {"target-sections", ck.Keys("chunking", "target_sections")},
{"max-section-tokens", ck.Keys("chunking", "max_section_tokens")}, {"min-section-tokens", ck.Keys("chunking", "min_section_tokens")},
{"normalize-max-segment-tokens", ck.Keys("normalization", "max_segment_tokens")},
} {
name, path := item.name, item.path
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*int)(nil), strconv.Atoi,
func(in *processInput, v commandkit.Supplied[int]) {
if v.Set {
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
}
}))
}
for _, item := range []struct {
name string
path ck.Path
}{
{"normalize-max-segment-gap", ck.Keys("normalization", "max_segment_gap")}, {"normalize-ellipsis-gap", ck.Keys("normalization", "ellipsis_gap")},
{"normalize-max-segment-duration", ck.Keys("normalization", "max_segment_duration")}, {"glossary-confidence-threshold", ck.Keys("thresholds", "glossary")},
{"homophones-confidence-threshold", ck.Keys("thresholds", "homophones")}, {"spoken-word-confidence-threshold", ck.Keys("thresholds", "spoken_word")},
{"grammar-confidence-threshold", ck.Keys("thresholds", "grammar")},
} {
name, path := item.name, item.path
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "Override application setting."}, (*float64)(nil), func(s string) (float64, error) { return strconv.ParseFloat(s, 64) },
func(in *processInput, v commandkit.Supplied[float64]) {
if v.Set {
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: v.Value, Flag: "--" + name})
}
}))
}
for _, item := range []struct {
name string
path ck.Path
}{
{"glossaries", ck.Keys("glossaries")}, {"protected-terms", ck.Keys("protected_terms")},
} {
name, path := item.name, item.path
flags = append(flags, commandkit.ValueFlag(commandkit.FlagMeta{Name: name, Usage: "JSON array of collection names, or null."}, (*any)(nil), parseCollectionNames,
func(in *processInput, value commandkit.Supplied[any]) {
if value.Set {
in.Overrides = append(in.Overrides, ck.Override{Path: path, Value: value.Value, Flag: "--" + name})
}
}))
}
flags = append(flags, commandkit.BoolFlag(commandkit.FlagMeta{Name: "reporting", Usage: "Enable ordinary reports."}, false,
func(in *processInput, v commandkit.Supplied[bool]) {
if v.Set {
in.Overrides = append(in.Overrides, ck.Override{Path: ck.Keys("reporting", "enabled"), Value: v.Value, Flag: "--reporting"})
}
}))
return commandkit.NewCommand(commandkit.CommandSpec[processInput]{Name: "process", Summary: "Polish a transcript.", Flags: flags, Run: runProcess,
Arguments: []commandkit.Argument[processInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "transcript", Usage: "Transcript JSON file."}, true, stringValue, func(in *processInput, v commandkit.Supplied[string]) { in.Transcript = v.Value })},
})
}
func parseCollectionNames(raw string) (any, error) {
value, err := commandkit.ParseCLIJSON(raw)
if err != nil {
return nil, err
}
if value == nil {
return nil, nil
}
names, ok := value.([]any)
if !ok {
return nil, fmt.Errorf("expected a JSON array of collection names or null")
}
for _, name := range names {
if _, ok := name.(string); !ok {
return nil, fmt.Errorf("collection names must be strings")
}
}
return names, nil
}

View File

@@ -0,0 +1,157 @@
package cli
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
func TestProcessShowsConfigurationLoadDiagnostics(t *testing.T) {
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name, config, finding string
}{
{"obsolete version", "version: 2\n", "docs/migrations/v3.md"},
{"invalid retry budget", "version: 3\nllm: {transient_retries: -1}\n", "retry budget must be non-negative"},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml")
if err := os.WriteFile(configPath, []byte(tc.config), 0600); err != nil {
t.Fatal(err)
}
output := filepath.Join(dir, "out.json")
var out, stderr bytes.Buffer
args := []string{"process", "--quiet", "--config", configPath, "--output", output, filepath.Join(dir, "transcript.json")}
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
var loadErr *ck.LoadError
if result.Status != 2 || !errors.As(result.Cause, &loadErr) || out.Len() != 0 ||
!strings.Contains(stderr.String(), tc.finding) || !strings.Contains(stderr.String(), configPath) {
t.Fatalf("status=%d cause=%v out=%q err=%q", result.Status, result.Cause, out.String(), stderr.String())
}
entries, err := os.ReadDir(dir)
if err != nil || len(entries) != 1 {
t.Fatalf("outputs created on load failure: %v, err=%v", entries, err)
}
})
}
}
func TestProcessUsesResolvedReportingMode(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
"blocked": "file",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
flags []string
status int
stdout bool
info bool
warning bool
}{
{"ordinary", nil, 0, true, false, false},
{"verbose", []string{"--verbose"}, 0, true, true, false},
{"quiet stdout", []string{"--quiet"}, 2, false, false, false},
{"quiet file", []string{"--quiet", "--output", filepath.Join(dir, "quiet.json")}, 0, false, false, false},
{"verbose after quiet", []string{"--quiet", "--verbose"}, 0, true, true, false},
{"quiet after verbose", []string{"--verbose", "--quiet", "--output", filepath.Join(dir, "ordered.json")}, 0, false, false, false},
{"debug disabled", []string{"--debug=false", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, false, false},
{"debug failure", []string{"--debug", "--work-dir", filepath.Join(dir, "blocked")}, 0, true, true, true},
{"debug quiet", []string{"--debug", "--quiet", "--output", filepath.Join(dir, "debug-quiet.json"), "--work-dir", filepath.Join(dir, "blocked")}, 0, false, false, false},
} {
t.Run(tc.name, func(t *testing.T) {
var out, stderr bytes.Buffer
args := []string{"process", "--config", filepath.Join(dir, "config.yml")}
args = append(args, tc.flags...)
args = append(args, filepath.Join(dir, "transcript.json"))
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
if result.Status != tc.status || (out.Len() > 0) != tc.stdout || strings.Contains(stderr.String(), "Completed:") != tc.info || strings.Contains(stderr.String(), "Debug artifact:") != tc.warning {
t.Fatalf("status=%d out=%q err=%q", result.Status, out.String(), stderr.String())
}
})
}
}
func TestProcessChecksOnlyTheActiveReportDirectory(t *testing.T) {
for _, tc := range []struct {
name string
flags []string
wantReport bool
wantError bool
}{
{"file output ignores blank configured directory", nil, true, false},
{"explicit blank directory", []string{"--report-dir="}, false, true},
{"disabled reporting ignores explicit blank directory", []string{"--reporting=false", "--report-dir="}, false, false},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml")
if err := os.WriteFile(configPath, []byte("version: 3\npipeline: {stages: []}\nreporting: {stdout_dir: ''}\n"), 0600); err != nil {
t.Fatal(err)
}
transcript := filepath.Join(dir, "transcript.json")
if err := os.WriteFile(transcript, []byte(`[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`), 0600); err != nil {
t.Fatal(err)
}
output := filepath.Join(dir, "output.json")
args := append([]string{"process", "--config", configPath, "--output", output}, tc.flags...)
var stdout, stderr bytes.Buffer
status := Run(context.Background(), append(args, transcript), &stdout, &stderr)
if (status != 0) != tc.wantError || (tc.wantError && !strings.Contains(stderr.String(), "report directory is required")) {
t.Fatalf("status=%d stdout=%q stderr=%q", status, stdout.String(), stderr.String())
}
_, err := os.Stat(filepath.Join(dir, "errors.yml"))
if (tc.wantReport && err != nil) || (!tc.wantReport && !os.IsNotExist(err)) {
t.Fatalf("report file: %v, want present=%t", err, tc.wantReport)
}
})
}
}
func TestProcessCollectionFlags(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\nglossary_collections:\n party: {entries: []}\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
app, err := buildApplication()
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
value string
status int
}{{`["party"]`, 0}, {`[]`, 0}, {`null`, 0}, {`[1]`, 2}, {`"party"`, 2}} {
var out, stderr bytes.Buffer
args := []string{"process", "--config", filepath.Join(dir, "config.yml"), "--glossaries", tc.value, filepath.Join(dir, "transcript.json")}
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
if outcome.Status != tc.status {
t.Fatalf("value=%q status=%d out=%q err=%q", tc.value, outcome.Status, out.String(), stderr.String())
}
}
}

View File

@@ -0,0 +1,160 @@
package cli
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"path/filepath"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
coreio "gitea.maximumdirect.net/eric/audita/v3/internal/core/io"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/preflight"
"go.maximumdirect.net/ckconfig"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
type processInputs struct {
Configuration *ck.Result[config.Config]
Runtime *llm.Runtime
Source *schema.SourceTranscript
RawTranscript []byte
ProtectedPaths []string
}
func loadProcessInputs(ctx context.Context, input processInput, mode commandkit.ReportingMode) (*processInputs, error) {
if mode == commandkit.ReportingModeQuiet && input.Output.Value == "" {
return nil, fmt.Errorf("quiet processing requires an output file or /dev/null")
}
selection := ckconfig.Selection{Config: input.Config, Overlay: input.Overlay, Overrides: input.Overrides}
loaded, err := loadApplication(ctx, selection, struct{}{}, nil)
if err != nil {
return nil, err
}
runtime, err := llm.NewRuntime(ctx, loaded, nil)
if err != nil {
return nil, err
}
raw, err := coreio.ReadRequiredFile(input.Transcript, "transcript")
if err != nil {
return nil, err
}
source, err := schema.ParseSourceTranscriptJSON(raw)
if err != nil {
return nil, err
}
protected := append([]string(nil), runtime.SourcePaths...)
for _, entry := range loaded.Sources() {
if entry.Kind == ck.SourceKindFile && entry.Path != "" {
protected = append(protected, entry.Path)
}
}
return &processInputs{Configuration: loaded, Runtime: runtime, Source: source, RawTranscript: raw, ProtectedPaths: protected}, nil
}
// preparedProcess groups the output resources and the facts needed for reports.
type preparedProcess struct {
Prepared *preflight.Prepared
RunID string
Started time.Time
}
func prepareProcess(ctx context.Context, input processInput, loaded *processInputs, stdout io.Writer) (*preparedProcess, error) {
cfg := &loaded.Configuration.Config
reportDir := ""
reportDirSet := false
for _, override := range input.Overrides {
if override.Path.String() == "/reporting/stdout_dir" {
reportDirSet = true
}
}
if cfg.Reporting.Enabled {
reportDir = cfg.Reporting.StdoutDir
if input.Output.Value != "" && !reportDirSet {
reportDir = filepath.Dir(input.Output.Value)
}
}
runID, err := generateRunID()
if err != nil {
return nil, err
}
started := time.Now().UTC()
var secrets []string
if cfg.Reporting.Enabled || input.Debug.Value {
secrets = loaded.Runtime.ConfiguredSecrets(ctx)
}
var initial diagnostics.ReportBundle
var failed func(error, []diagnostics.FileFailure) diagnostics.ReportBundle
if cfg.Reporting.Enabled {
snapshot := reporting.Snapshot{RunID: runID, Status: reporting.StatusInProgress, StartedAt: started, TranscriptPath: input.Transcript, OutputPath: input.Output.Value}
for index, item := range cfg.Pipeline.Stages {
snapshot.StageNames = append(snapshot.StageNames, reporting.StageLabel(index, item.Module, item.ID))
}
initial, err = renderReportBundle(snapshot, secrets...)
if err != nil {
return nil, err
}
failed = func(primary error, failures []diagnostics.FileFailure) diagnostics.ReportBundle {
failedSnapshot := snapshot
completed := time.Now().UTC()
failedSnapshot.CompletedAt = &completed
if primary != nil {
failedSnapshot.Errors = []reporting.ErrorRecord{{Scope: "run", Phase: "initialization", Reason: "canceled", Message: "Processing was canceled."}}
}
bundle, _ := renderReportBundle(failedOutputSnapshot(failedSnapshot, failures), secrets...)
return bundle
}
}
var debug *preflight.DebugInput
var warnings []error
if input.Debug.Value {
var effective bytes.Buffer
if err := loaded.Configuration.WriteEffective(&effective); err != nil {
warnings = append(warnings, err)
}
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
}
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
if err != nil {
return err
}
if profile.Backend != nil {
_, err = loaded.Runtime.Engine.InspectBackend(ctx, *profile.Backend)
}
return err
}, Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
if err != nil {
return nil, err
}
prepared.Warnings = append(warnings, prepared.Warnings...)
return &preparedProcess{Prepared: prepared, RunID: runID, Started: started}, nil
}
func generateRunID() (string, error) {
var random [12]byte
if _, err := rand.Read(random[:]); err != nil {
return "", err
}
return "run-" + hex.EncodeToString(random[:]), nil
}
func renderReportBundle(value reporting.Snapshot, secrets ...string) (diagnostics.ReportBundle, error) {
errorsYAML, err := reporting.YAML(value.ErrorsDocument(), secrets...)
if err != nil {
return diagnostics.ReportBundle{}, err
}
recoveriesYAML, err := reporting.YAML(value.RecoveriesDocument(), secrets...)
if err != nil {
return diagnostics.ReportBundle{}, err
}
return diagnostics.ReportBundle{Errors: errorsYAML, Recoveries: recoveriesYAML, Markdown: reporting.Markdown(value, secrets...)}, nil
}

View File

@@ -0,0 +1,108 @@
package cli
import (
"context"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
func TestQuietDestinationCheckPrecedesInputAcquisition(t *testing.T) {
_, err := loadProcessInputs(context.Background(), processInput{Transcript: "missing.json"}, commandkit.ReportingModeQuiet)
if err == nil || !strings.Contains(err.Error(), "/dev/null") {
t.Fatalf("error=%v", err)
}
}
func TestProcessAcquisitionCapturesConsumedFiles(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
"prompts.yml": "prompts:\n - id: extra\n messages:\n - role: user\n content_file: text.txt\n output:\n format: text\n validation_mode: none\n",
"text.txt": "example",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
result, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
if err != nil {
t.Fatal(err)
}
for _, name := range []string{"config.yml", "prompts.yml", "text.txt"} {
if !slices.Contains(result.ProtectedPaths, filepath.Join(dir, name)) {
t.Errorf("missing %s in %v", name, result.ProtectedPaths)
}
}
}
func TestPrepareProcessInitializesReportsAndToleratesDebugLoss(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
"blocked": "not a directory",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json"), Output: commandkit.Supplied[string]{Value: filepath.Join(dir, "output.json"), Set: true}, Debug: commandkit.Supplied[bool]{Value: true, Set: true}, Overrides: []ck.Override{{Path: ck.Keys("diagnostics", "work_dir"), Value: filepath.Join(dir, "blocked"), Flag: "--work-dir"}}}
loaded, err := loadProcessInputs(context.Background(), input, commandkit.ReportingModeOrdinary)
if err != nil {
t.Fatal(err)
}
prepared, err := prepareProcess(context.Background(), input, loaded, io.Discard)
if err != nil {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
if len(prepared.Prepared.Warnings) == 0 {
t.Fatal("debug loss was not reported")
}
report, err := os.ReadFile(filepath.Join(dir, "errors.yml"))
if err != nil || !strings.Contains(string(report), "in_progress") {
t.Fatalf("report=%q err=%v", report, err)
}
if _, err := os.Stat(input.Output.Value); !os.IsNotExist(err) {
t.Fatalf("output published early: %v", err)
}
}
func TestCollectionFlagJSON(t *testing.T) {
for _, tc := range []struct {
raw string
valid bool
null bool
size int
}{
{`["party","spells"]`, true, false, 2}, {`[]`, true, false, 0}, {`null`, true, true, 0},
{`"party"`, false, false, 0}, {`[1]`, false, false, 0}, {`["x"] true`, false, false, 0},
} {
result, err := parseCollectionNames(tc.raw)
if (err == nil) != tc.valid {
t.Fatalf("%s error=%v", tc.raw, err)
}
if !tc.valid {
continue
}
if tc.null {
if result != nil {
t.Fatalf("null=%v", result)
}
continue
}
list, ok := result.([]any)
if !ok || len(list) != tc.size {
t.Fatalf("%s result=%v", tc.raw, result)
}
}
}

View File

@@ -0,0 +1,200 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
commandkit "go.maximumdirect.net/commandkit"
ck "go.maximumdirect.net/configkit"
)
type controlledTransport func(*http.Request) (*http.Response, error)
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
var mu sync.Mutex
var models []string
var prompts []string
proposalCalls := 0
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
var request struct {
Model string `json:"model"`
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
return nil, err
}
var contents string
for _, message := range request.Messages {
contents += message.Content
}
mu.Lock()
models = append(models, request.Model)
prompts = append(prompts, contents)
if request.Model == "model-one" {
proposalCalls++
}
firstProposalCall := proposalCalls
mu.Unlock()
result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}`
switch request.Model {
case "model-one":
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}`
if firstProposalCall == 1 {
result = `{"invalid":true}`
}
case "model-two":
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}`
case "model-three":
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast","confidence":1}]}`
}
encoded, _ := json.Marshal(result)
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
files := map[string]string{
"config.yml": `version: 3
glossary_collections:
party:
entries: [{id: alice, name: Alice, category: person}]
spells:
entries: [{id: shield, name: Shield, category: spell}]
protected_terms: [party, spells]
llm: {validation_profile: review}
pipeline:
stages:
- module: glossary
id: "party/../|<tag>"
glossaries: [party]
llm: {proposal_profile: one}
- module: glossary
id: spell
glossaries: [spells]
llm: {proposal_profile: two}
- module: grammar
glossaries: [party, spells]
llm: {proposal_profile: three}
`,
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
}
for name, value := range files {
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
if err != nil {
t.Fatal(err)
}
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
loaded.Config.Diagnostics.WorkDir = dir
loaded.Config.Reporting.StdoutDir = dir
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json"), Debug: commandkit.Supplied[bool]{Value: true}}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
if err != nil {
t.Fatal(err)
}
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
}
if len(prompts) > 4 && !strings.Contains(prompts[4], "Alice casts Sheeld.") {
t.Fatalf("second proposal missed earlier edit: %s", prompts[4])
}
if len(result.Snapshot.WorkUnits) != 3 || result.Snapshot.WorkUnits[0].StageID == nil || *result.Snapshot.WorkUnits[0].StageID != "party/../|<tag>" || result.Snapshot.WorkUnits[1].StageIndex != 1 || result.Snapshot.WorkUnits[2].StageID != nil {
t.Fatalf("work unit identities: %+v", result.Snapshot.WorkUnits)
}
if len(result.Snapshot.Recoveries) != 1 || result.Snapshot.Recoveries[0].StageIndex != 0 || result.Snapshot.Recoveries[0].StageID == nil || *result.Snapshot.Recoveries[0].StageID != "party/../|<tag>" {
t.Fatalf("recovery attribution: %+v", result.Snapshot.Recoveries)
}
markdown := string(reporting.Markdown(result.Snapshot))
if !strings.Contains(markdown, `1: glossary (party/../\|&lt;tag&gt;)`) || !strings.Contains(markdown, "2: glossary (spell)") || !strings.Contains(markdown, "3: grammar") {
t.Fatalf("report labels: %s", markdown)
}
for _, artifact := range []string{diagnostics.ArtifactCorrectionLedger, diagnostics.ArtifactLLMExecution} {
raw, readErr := os.ReadFile(filepath.Join(prepared.DebugPath(), artifact))
if readErr != nil {
t.Fatal(readErr)
}
if !strings.Contains(string(raw), `"stage_index": 0`) || !strings.Contains(string(raw), `"stage_id": "party/../|\u003ctag\u003e"`) || !strings.Contains(string(raw), `"stage_index": 1`) || !strings.Contains(string(raw), `"stage_id": "spell"`) || !strings.Contains(string(raw), `"stage_index": 2`) {
t.Fatalf("%s missing attribution: %s", artifact, raw)
}
}
if !reflect.DeepEqual(models, []string{"model-one", "model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
t.Fatalf("models=%v", models)
}
}
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
dir := t.TempDir()
files := map[string]string{
"config.yml": "version: 3\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar, id: failing}]\n",
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
}
for name, value := range files {
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
})
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
if err != nil {
t.Fatal(err)
}
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
loaded.Config.Reporting.StdoutDir = dir
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
if err != nil {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
if err != nil {
t.Fatal(err)
}
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" || result.Snapshot.Errors[0].StageID == nil || *result.Snapshot.Errors[0].StageID != "failing" {
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
}
}

View File

@@ -0,0 +1,113 @@
package cli
import (
"context"
"errors"
"fmt"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/redaction"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
commandkit "go.maximumdirect.net/commandkit"
)
// finishProcess delivers the transcript and reports before emitting success information.
func finishProcess(inv commandkit.Invocation, input processInput, loaded *processInputs, prepared *preparedProcess, snapshot reporting.Snapshot, transcript []byte, primary error) commandkit.Outcome {
reporter := inv.Reporter()
secrets := loaded.Runtime.ConfiguredSecrets(inv.Context)
safe := redaction.New(secrets)
for _, warning := range prepared.Prepared.Warnings {
_ = reporter.Warn("Debug artifact: " + safe.String(warning.Error()))
}
completed := time.Now().UTC()
snapshot.RunID, snapshot.StartedAt, snapshot.CompletedAt = prepared.RunID, prepared.Started, &completed
snapshot.TranscriptPath, snapshot.OutputPath = input.Transcript, input.Output.Value
if primary == nil {
primary = inv.Context.Err()
}
snapshot.Status = reporting.StatusCompleted
if primary != nil {
snapshot = failedProcessingSnapshot(snapshot, primary, inv.Context.Err())
}
reportPaths := prepared.Prepared.Output.ReportPaths()
var reports diagnostics.ReportBundle
var failed func([]diagnostics.FileFailure) diagnostics.ReportBundle
if len(reportPaths) > 0 {
var err error
reports, err = renderReportBundle(snapshot, secrets...)
if err != nil {
if primary == nil {
primary = err
}
snapshot.Status = reporting.StatusFailed
}
failed = func(failures []diagnostics.FileFailure) diagnostics.ReportBundle {
result, bundleErr := renderReportBundle(failedOutputSnapshot(snapshot, failures), secrets...)
if bundleErr != nil {
return reports
}
return result
}
}
if canceled := inv.Context.Err(); primary == nil && canceled != nil {
primary = canceled
snapshot = failedProcessingSnapshot(snapshot, canceled, canceled)
if failed != nil {
reports = failed(nil)
}
}
result := prepared.Prepared.Output.Finish(primary, transcript, reports, failed)
if result != nil {
return commandkit.Failure(safe.String(result.Error())).WithCause(result)
}
_ = reporter.AtEnd().Info(reporting.CompletionLine(snapshot))
for _, path := range reportPaths {
_ = reporter.AtEnd().Info(fmt.Sprintf("Report: %s", safe.String(path)))
}
if prepared.Prepared.Debug != nil {
_ = reporter.AtEnd().Info(fmt.Sprintf("Diagnostics: %s", safe.String(prepared.Prepared.Debug.Path())))
}
return commandkit.Success()
}
func failedProcessingSnapshot(snapshot reporting.Snapshot, primary, contextErr error) reporting.Snapshot {
for _, record := range snapshot.Errors {
if record.Scope == "run" {
return failedOutputSnapshot(snapshot, nil)
}
}
record := reporting.ErrorRecord{Scope: "run", Reason: "processing_failure", Message: "Processing failed"}
if errors.Is(primary, context.Canceled) || (contextErr != nil && errors.Is(primary, contextErr)) {
record.Phase = "processing"
record.Reason = string(outcomes.ReasonCanceled)
record.Message = "Processing was canceled."
}
snapshot.Errors = append(snapshot.Errors, record)
return failedOutputSnapshot(snapshot, nil)
}
// failedOutputSnapshot derives the count from the same run records sent to each report.
func failedOutputSnapshot(snapshot reporting.Snapshot, failures []diagnostics.FileFailure) reporting.Snapshot {
snapshot.Status = reporting.StatusFailed
if len(failures) > 0 {
snapshot.Errors = append([]reporting.ErrorRecord(nil), snapshot.Errors...)
}
for _, failure := range failures {
record := reporting.ErrorRecord{Scope: "run", Reason: "output_failure", Message: "Required output delivery failed", OutputPath: failure.Path}
if failure.Kind == "report initialization" {
record.Phase = "initialization"
record.Message = "Required report initialization failed"
}
snapshot.Errors = append(snapshot.Errors, record)
}
count := 0
for _, record := range snapshot.Errors {
if record.Scope == "run" {
count++
}
}
snapshot.Summary.RunErrors = &count
return snapshot
}

View File

@@ -0,0 +1,197 @@
package cli
import (
"bytes"
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
commandkit "go.maximumdirect.net/commandkit"
"gopkg.in/yaml.v3"
)
func TestPreparedDeliveryFollowsReportingMode(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\nreporting:\n enabled: false\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
for _, tc := range []struct {
name string
flags []string
file bool
wantInfo bool
}{
{"ordinary", nil, false, false}, {"verbose", []string{"--verbose"}, false, true}, {"quiet file", []string{"--quiet"}, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
if tc.file {
input.Output = commandkit.Supplied[string]{Value: filepath.Join(dir, "result.json"), Set: true}
}
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
if err != nil {
return commandkit.Usage(err.Error())
}
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil {
return commandkit.Failure(err.Error())
}
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {
t.Fatal(err)
}
var out, stderr bytes.Buffer
args := append([]string{"deliver"}, tc.flags...)
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &stderr})
if outcome.Status != 0 {
t.Fatalf("outcome=%+v err=%s", outcome, stderr.String())
}
if tc.file {
data, err := os.ReadFile(input.Output.Value)
if err != nil || string(data) != "transcript bytes" {
t.Fatalf("file=%q err=%v", data, err)
}
if out.Len() != 0 {
t.Fatalf("stdout=%q", out.String())
}
} else if out.String() != "transcript bytes" {
t.Fatalf("stdout=%q", out.String())
}
if strings.Contains(stderr.String(), "Completed:") != tc.wantInfo {
t.Fatalf("stderr=%q", stderr.String())
}
})
}
}
func TestDeliveryFailureSuppressesCompletionSummary(t *testing.T) {
dir := t.TempDir()
t.Chdir(dir)
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline:\n stages: []\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0644); err != nil {
t.Fatal(err)
}
}
input := processInput{Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true}, Transcript: filepath.Join(dir, "transcript.json")}
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
if err != nil {
return commandkit.Failure(err.Error())
}
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil {
return commandkit.Failure(err.Error())
}
return finishProcess(inv, input, loaded, prepared, reporting.Snapshot{}, []byte("transcript bytes"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {
t.Fatal(err)
}
var stderr bytes.Buffer
outcome := app.Execute(context.Background(), []string{"deliver", "--verbose"}, commandkit.Streams{Out: shortProcessWriter{}, Err: &stderr})
if outcome.Status == 0 || strings.Contains(stderr.String(), "Completed:") {
t.Fatalf("status=%d stderr=%q", outcome.Status, stderr.String())
}
var errs reporting.ErrorsDocument
var recoveries reporting.RecoveriesDocument
for path, target := range map[string]any{"errors.yml": &errs, "recoveries.yml": &recoveries} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if err := yaml.Unmarshal(data, target); err != nil {
t.Fatal(err)
}
}
if errs.Status != reporting.StatusFailed || recoveries.Status != reporting.StatusFailed ||
errs.Summary.RunErrors == nil || *errs.Summary.RunErrors != 1 ||
recoveries.Summary.RunErrors == nil || *recoveries.Summary.RunErrors != 1 ||
len(errs.Errors) != 1 || errs.Errors[0].Reason != "output_failure" {
t.Fatalf("failed output reports disagree: %+v %+v", errs, recoveries)
}
}
func TestCanceledEmptyPipelineLeavesTranscriptAndFinalizesReports(t *testing.T) {
dir := t.TempDir()
for name, contents := range map[string]string{
"config.yml": "version: 3\npipeline: {stages: []}\n",
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
"output.json": "previous transcript",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}
input := processInput{
Config: commandkit.Supplied[string]{Value: filepath.Join(dir, "config.yml"), Set: true},
Transcript: filepath.Join(dir, "transcript.json"),
Output: commandkit.Supplied[string]{Value: filepath.Join(dir, "output.json"), Set: true},
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cmd := commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "deliver", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome {
loaded, err := loadProcessInputs(inv.Context, input, inv.ReportingMode())
if err != nil {
t.Fatal(err)
}
prepared, err := prepareProcess(inv.Context, input, loaded, inv.Reporter().Output())
if err != nil {
t.Fatal(err)
}
defer prepared.Prepared.Output.Close()
result, err := executePipeline(inv.Context, loaded, prepared, sourceToCanonicalTranscript(loaded.Source), func(error) {})
if err != nil {
t.Fatal(err)
}
cancel()
return finishProcess(inv, input, loaded, prepared, result.Snapshot, []byte("new transcript"), nil)
}})
app, err := commandkit.New(commandkit.Identity{Name: "test"}, cmd)
if err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
outcome := app.Execute(ctx, []string{"deliver", "--verbose"}, commandkit.Streams{Out: &stdout, Err: &stderr})
if outcome.Status == 0 || !errors.Is(outcome.Cause, context.Canceled) || strings.Contains(stderr.String(), "Completed:") {
t.Fatalf("outcome=%+v stderr=%q", outcome, stderr.String())
}
if data, err := os.ReadFile(input.Output.Value); err != nil || string(data) != "previous transcript" {
t.Fatalf("transcript changed after cancellation: %q, %v", data, err)
}
var errorsDoc reporting.ErrorsDocument
var recoveries reporting.RecoveriesDocument
for name, target := range map[string]any{"errors.yml": &errorsDoc, "recoveries.yml": &recoveries} {
data, err := os.ReadFile(filepath.Join(dir, name))
if err != nil {
t.Fatal(err)
}
if err := yaml.Unmarshal(data, target); err != nil {
t.Fatal(err)
}
}
if errorsDoc.Status != reporting.StatusFailed || recoveries.Status != reporting.StatusFailed ||
len(errorsDoc.Errors) != 1 || errorsDoc.Errors[0].Reason != "canceled" {
t.Fatalf("reports did not capture cancellation: %+v %+v", errorsDoc, recoveries)
}
}
type shortProcessWriter struct{}
func (shortProcessWriter) Write(data []byte) (int, error) { return len(data) / 2, nil }

View File

@@ -0,0 +1,81 @@
package cli
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
cfg := &loaded.Configuration.Config
specs := contracts.ResolveModuleRunSpecs(cfg)
debug := prepared.Prepared.Debug
var observer *llm.ObservationCollector
var hooks llm.CallHooks
if debug != nil {
observer = llm.NewObservationCollector()
hooks = observer.Hooks()
}
for i := range specs {
item := &specs[i]
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
}
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
}
item.ProposalClient, item.ValidationClient = proposal, validation
if item.Settings.LLM.CorrectionRepairAttempts > 0 {
repair, err := loaded.Runtime.NewRoleClient("correction_repair", *item.Settings.LLM.CorrectionRepairProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s correction repair client: %w", item.InstanceName, err)
}
item.RepairClient = repair
item.CaptureProposalExchange = true
}
}
debugPath := prepared.DebugPath()
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
})
if debug != nil {
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(observer.Snapshot())))
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
}
var failures []processreport.RunFailure
if runErr != nil {
failure := processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure}
var cause *structuredoutput.Failure
if errors.As(runErr, &cause) {
failure.Reason = cause.Reason
} else if errors.Is(runErr, context.Canceled) || (ctx.Err() != nil && errors.Is(runErr, ctx.Err())) {
failure.Reason = outcomes.ReasonCanceled
}
if len(output.ModuleResults) > 0 {
last := output.ModuleResults[len(output.ModuleResults)-1]
if last.Status == runner.ModuleStatusFailed {
index := last.StageIndex
failure.Origin = &outcomes.Reference{StageIndex: &index, StageID: last.StageID, ModuleKey: last.ModuleKey, ModuleInstance: last.ModuleInstance}
}
}
failures = append(failures, failure)
}
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started,
ModuleSpecs: specs, RunOutput: &output, RunFailures: failures, SummaryOnly: !cfg.Reporting.Enabled})
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
}

View File

@@ -2,13 +2,11 @@ package cli
import (
"context"
"flag"
"fmt"
"io"
"gitea.maximumdirect.net/eric/ReleaseKit/buildinfo"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
commandkit "go.maximumdirect.net/commandkit"
)
func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transcript {
@@ -30,241 +28,12 @@ func sourceToCanonicalTranscript(source *schema.SourceTranscript) *schema.Transc
return &schema.Transcript{Segments: segments}
}
// Run executes the Audita CLI with the provided arguments and streams.
// Run executes the application with the provided arguments and streams.
func Run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
if len(args) == 0 {
writeRootUsage(stdout)
return 0
}
if len(args) == 2 && args[0] == "help" {
switch args[1] {
case "process":
writeProcessUsage(stdout)
return 0
case "config":
writeConfigUsage(stdout)
return 0
case "version":
writeVersionUsage(stdout)
return 0
}
}
if isHelpCommand(args) {
writeRootUsage(stdout)
return 0
}
if args[0] == "process" {
return runProcess(ctx, args[1:], stdout, stderr)
}
if args[0] == "config" {
return runConfig(ctx, args[1:], stdout, stderr)
}
if args[0] == "version" {
return runVersion(args[1:], stdout, stderr)
}
fmt.Fprintf(stderr, "audita: unknown command %q\n\n", args[0])
writeRootUsage(stderr)
return 2
}
func runVersion(args []string, stdout, stderr io.Writer) int {
flags := flag.NewFlagSet("version", flag.ContinueOnError)
flags.SetOutput(stderr)
flags.Usage = func() {} // Route usage after parsing so help stays on stdout.
if err := flags.Parse(args); err != nil {
if err == flag.ErrHelp {
writeVersionUsage(stdout)
return 0
}
writeVersionUsage(stderr)
return 2
}
if positional := flags.Args(); len(positional) > 0 {
fmt.Fprintf(stderr, "audita version: unexpected argument %q\n\n", positional[0])
writeVersionUsage(stderr)
return 2
}
fmt.Fprintf(stdout, "audita %s\n", buildinfo.Version())
return 0
}
func runConfig(ctx context.Context, args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || isHelpCommand(args) || args[0] == "--help" || args[0] == "-h" {
writeConfigUsage(stdout)
return 0
}
switch args[0] {
case "validate":
return runConfigValidate(ctx, args[1:], stdout, stderr)
case "print-effective":
return runConfigPrintEffective(ctx, args[1:], stdout, stderr)
default:
fmt.Fprintf(stderr, "audita config: unknown command %q\n\n", args[0])
writeConfigUsage(stderr)
return 2
}
}
func runConfigValidate(ctx context.Context, args []string, stdout, stderr io.Writer) int {
input, help, err := parseConfigInput("validate", args, stderr)
app, err := buildApplication()
if err != nil {
fmt.Fprintf(stderr, "audita config validate: %s\n", configInputError(err))
return 2
}
if help {
writeConfigValidateUsage(stdout)
return 0
}
selected, err := loadConfigInput(input, "validate")
if err != nil {
fmt.Fprintf(stderr, "audita config validate: %s\n", configInputError(err))
return 2
}
if _, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil); err != nil {
fmt.Fprintf(stderr, "audita config validate: %s\n", llm.RedactError(err, selected.Result.Config))
if ctx.Err() != nil || llm.IsAssetError(err) {
return 1
}
return 2
}
fmt.Fprintln(stdout, "config is valid")
return 0
}
func runConfigPrintEffective(ctx context.Context, args []string, stdout, stderr io.Writer) int {
input, help, err := parseConfigInput("print-effective", args, stderr)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %s\n", configInputError(err))
return 2
}
if help {
writeConfigPrintEffectiveUsage(stdout)
return 0
}
selected, err := loadConfigInput(input, "print-effective")
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %s\n", configInputError(err))
return 2
}
llmRuntime, err := llm.NewRuntime(ctx, selected.Result.Config, selected.Path, nil)
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %s\n", llm.RedactError(err, selected.Result.Config))
if ctx.Err() != nil || llm.IsAssetError(err) {
return 1
}
return 2
}
if input.View == "settings" {
err = selected.Result.WriteEffective(stdout)
} else {
var output []byte
if input.View == "profiles" {
output, err = llmRuntime.SelectedProfilesYAML()
} else {
output, err = llm.ExplanationYAML(selected.Result)
}
if err == nil {
_, err = stdout.Write(output)
}
}
if err != nil {
fmt.Fprintf(stderr, "audita config print-effective: %v\n", err)
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
func isHelpCommand(args []string) bool {
if len(args) == 0 {
return false
}
if len(args) == 1 {
switch args[0] {
case "help", "-h", "--help":
return true
}
}
if len(args) == 2 && args[0] == "help" {
switch args[1] {
case "process":
return true
}
}
return false
}
func writeRootUsage(w io.Writer) {
fmt.Fprintln(w, "Audita is a transcript processing CLI.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita <command> [options]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " process Process a transcript JSON file")
fmt.Fprintln(w, " config Validate and inspect config")
fmt.Fprintln(w, " version Print the Audita version")
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
}
func writeVersionUsage(w io.Writer) {
fmt.Fprintln(w, "Print the Audita version.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita version")
}
func writeConfigUsage(w io.Writer) {
fmt.Fprintln(w, "Validate and inspect Audita config.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config <command> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Commands:")
fmt.Fprintln(w, " validate Validate a versioned YAML config file")
fmt.Fprintln(w, " print-effective Print redacted effective settings YAML")
}
func writeConfigValidateUsage(w io.Writer) {
fmt.Fprintln(w, "Validate a versioned YAML config file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config validate --config <path> [--overlay <name>] [setting flags]")
}
func writeConfigPrintEffectiveUsage(w io.Writer) {
fmt.Fprintln(w, "Print redacted effective settings YAML.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita config print-effective [--config <path>] [--overlay <name>] [setting flags]")
}
func writeProcessUsage(w io.Writer) {
fmt.Fprintln(w, "Process a transcript JSON file.")
fmt.Fprintln(w)
fmt.Fprintln(w, "Usage:")
fmt.Fprintln(w, " audita process <transcript.json> [flags]")
fmt.Fprintln(w)
fmt.Fprintln(w, "Flags:")
for _, name := range []string{
"config", "overlay", "glossary", "output",
"proposal-profile", "validation-profile", "modules", "output-schema",
"validation-max-prompt-tokens", "target-sections", "max-section-tokens", "min-section-tokens",
"normalize-max-segment-gap", "normalize-ellipsis-gap", "normalize-max-segment-duration", "normalize-max-segment-tokens",
"glossary-confidence-threshold", "homophones-confidence-threshold", "spoken-word-confidence-threshold", "grammar-confidence-threshold",
"transcript-description", "work-dir",
} {
fmt.Fprintf(w, " --%s\n", name)
}
fmt.Fprintln(w, " --reporting[=true|false] Write the three-file report bundle (default true)")
fmt.Fprintln(w, " --report-dir DIR Select the report directory (stdout setting for stdout; output parent by default for files)")
fmt.Fprintln(w, " --debug[=true|false] Persist detailed diagnostics (default false)")
fmt.Fprintln(w)
fmt.Fprintln(w, "Example:")
fmt.Fprintln(w, " audita process transcript.json --glossary glossary.yaml --output corrected.json")
return app.Execute(ctx, args, commandkit.Streams{Out: stdout, Err: stderr}).Status
}

View File

@@ -12,65 +12,42 @@ import (
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
t.Helper()
dir := t.TempDir()
profiles := filepath.Join(dir, "profiles")
if err := os.Mkdir(profiles, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
t.Fatal(err)
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": operatorDocument} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
return main
}
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
definitions := map[string]string{
"malformed operator YAML": "profiles:\n proposal: [\n",
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
"malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
"missing operator message resource": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content_file: missing-instruction.md\n output:\n format: text\n validation_mode: none\n",
}
commands := []struct {
name string
args []string
}{
{name: "validate", args: []string{"config", "validate"}},
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
}
for definition, document := range definitions {
for _, command := range commands {
t.Run(definition+"/"+command.name, func(t *testing.T) {
main := writeRuntimeErrorConfig(t, document)
args := append(append([]string(nil), command.args...), "--config", main)
var stdout, stderr bytes.Buffer
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatalf("configured secret leaked: %q", stderr.String())
}
})
}
for name, document := range definitions {
t.Run(name, func(t *testing.T) {
main := writeRuntimeErrorConfig(t, document)
var stdout, stderr bytes.Buffer
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatal("configured secret leaked")
}
})
}
}
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
commands := [][]string{
{"config", "validate", "--config", main},
{"config", "print-effective", "--config", main, "--view", "profiles"},
}
for _, args := range commands {
ctx, cancel := context.WithCancel(context.Background())
cancel()
var stdout, stderr bytes.Buffer
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
}
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
ctx, cancel := context.WithCancel(context.Background())
cancel()
var stdout, stderr bytes.Buffer
if code := Run(ctx, []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("cancellation succeeded: stdout=%q stderr=%q", stdout.String(), stderr.String())
}
}

View File

@@ -7,7 +7,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/ReleaseKit/buildinfo"
"go.maximumdirect.net/releasekit/buildinfo"
)
func TestVersionCommandRunsWithoutApplicationSetup(t *testing.T) {
@@ -30,12 +30,10 @@ func TestVersionCommandHelp(t *testing.T) {
args []string
want string
}{
{name: "root help", args: []string{"--help"}, want: "version Print the Audita version"},
{name: "root help", args: []string{"--help"}, want: "version Show application or CommandKit version."},
{name: "help version", args: []string{"help", "version"}, want: "audita version"},
{name: "short flag", args: []string{"version", "-h"}, want: "audita version"},
{name: "long flag", args: []string{"version", "--help"}, want: "audita version"},
{name: "parser help", args: []string{"version", "--h"}, want: "audita version"},
{name: "help short circuits", args: []string{"version", "-h", "--help", "extra"}, want: "audita version"},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer

View File

@@ -3,7 +3,7 @@ package chunking
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
// Section represents a contiguous chunk of transcript segments with metadata.

View File

@@ -4,7 +4,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
// mapTokenEstimator provides deterministic per-segment token counts for tests.

View File

@@ -1,84 +0,0 @@
package chunking
// Summary provides a concise overview of chunking results for reports
type Summary struct {
ChunkCount int `json:"chunk_count"`
MinEstimatedTokens int `json:"min_estimated_chunk_tokens"`
MaxEstimatedTokens int `json:"max_estimated_chunk_tokens"`
TotalEstimatedTokens int `json:"total_estimated_transcript_tokens"`
TargetSections *int `json:"target_sections,omitempty"`
MaxSectionTokens int `json:"max_section_tokens"`
MinSectionTokens int `json:"min_section_tokens"`
}
// ChunkSummary represents a single chunk's metadata for diagnostics
type ChunkSummary struct {
Index int `json:"index"`
StartSegmentID int `json:"start_segment_id"`
EndSegmentID int `json:"end_segment_id"`
EstimatedTokens int `json:"estimated_tokens"`
SegmentCount int `json:"segment_count"`
}
// DetailedSummary provides per-chunk details for diagnostics
type DetailedSummary struct {
Summary `json:",inline"`
Chunks []ChunkSummary `json:"chunks"`
}
// ComputeSummary creates a Summary from sections and config
func ComputeSummary(sections []Section, config ChunkingConfig) Summary {
if len(sections) == 0 {
return Summary{
ChunkCount: 0,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
TargetSections: config.TargetSections,
}
}
minTokens := sections[0].EstimatedTokens
maxTokens := sections[0].EstimatedTokens
totalTokens := 0
for _, sec := range sections {
if sec.EstimatedTokens < minTokens {
minTokens = sec.EstimatedTokens
}
if sec.EstimatedTokens > maxTokens {
maxTokens = sec.EstimatedTokens
}
totalTokens += sec.EstimatedTokens
}
return Summary{
ChunkCount: len(sections),
MinEstimatedTokens: minTokens,
MaxEstimatedTokens: maxTokens,
TotalEstimatedTokens: totalTokens,
TargetSections: config.TargetSections,
MaxSectionTokens: config.MaxSectionTokens,
MinSectionTokens: config.MinSectionTokens,
}
}
// ComputeDetailedSummary creates a DetailedSummary from sections and config
func ComputeDetailedSummary(sections []Section, config ChunkingConfig) DetailedSummary {
summary := ComputeSummary(sections, config)
chunks := make([]ChunkSummary, len(sections))
for i, sec := range sections {
chunks[i] = ChunkSummary{
Index: sec.Index,
StartSegmentID: sec.StartSegmentID,
EndSegmentID: sec.EndSegmentID,
EstimatedTokens: sec.EstimatedTokens,
SegmentCount: len(sec.Segments),
}
}
return DetailedSummary{
Summary: summary,
Chunks: chunks,
}
}

View File

@@ -1,148 +0,0 @@
package chunking
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestComputeSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
{Index: 1, EstimatedTokens: 50, StartSegmentID: 3, EndSegmentID: 4},
{Index: 2, EstimatedTokens: 20, StartSegmentID: 5, EndSegmentID: 5},
}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 3 {
t.Errorf("expected chunk_count=3, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 20 {
t.Errorf("expected min_estimated_tokens=20, got %d", summary.MinEstimatedTokens)
}
if summary.MaxEstimatedTokens != 50 {
t.Errorf("expected max_estimated_tokens=50, got %d", summary.MaxEstimatedTokens)
}
if summary.TotalEstimatedTokens != 100 {
t.Errorf("expected total_estimated_tokens=100, got %d", summary.TotalEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens=100, got %d", summary.MaxSectionTokens)
}
if summary.MinSectionTokens != 10 {
t.Errorf("expected min_section_tokens=10, got %d", summary.MinSectionTokens)
}
if summary.TargetSections != nil {
t.Errorf("expected target_sections=nil, got %v", summary.TargetSections)
}
}
func TestComputeSummaryWithTarget(t *testing.T) {
target := 5
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
TargetSections: &target,
}
sections := []Section{
{Index: 0, EstimatedTokens: 30, StartSegmentID: 1, EndSegmentID: 2},
}
summary := ComputeSummary(sections, config)
if summary.TargetSections == nil || *summary.TargetSections != 5 {
t.Errorf("expected target_sections=5, got %v", summary.TargetSections)
}
}
func TestComputeSummaryEmptySections(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{}
summary := ComputeSummary(sections, config)
if summary.ChunkCount != 0 {
t.Errorf("expected chunk_count=0, got %d", summary.ChunkCount)
}
if summary.MinEstimatedTokens != 0 {
t.Errorf("expected min_estimated_tokens=0 for empty, got %d", summary.MinEstimatedTokens)
}
if summary.MaxSectionTokens != 100 {
t.Errorf("expected max_section_tokens preserved, got %d", summary.MaxSectionTokens)
}
}
func TestComputeDetailedSummary(t *testing.T) {
config := ChunkingConfig{
MaxSectionTokens: 100,
MinSectionTokens: 10,
}
sections := []Section{
{
Index: 0,
EstimatedTokens: 30,
StartSegmentID: 1,
EndSegmentID: 2,
Segments: make([]schema.Segment, 2), // 2 segments
},
{
Index: 1,
EstimatedTokens: 50,
StartSegmentID: 3,
EndSegmentID: 5,
Segments: make([]schema.Segment, 3), // 3 segments
},
}
detailed := ComputeDetailedSummary(sections, config)
if detailed.ChunkCount != 2 {
t.Errorf("expected chunk_count=2, got %d", detailed.ChunkCount)
}
if len(detailed.Chunks) != 2 {
t.Fatalf("expected 2 chunk entries, got %d", len(detailed.Chunks))
}
// Check first chunk
if detailed.Chunks[0].Index != 0 {
t.Errorf("expected chunk[0].index=0, got %d", detailed.Chunks[0].Index)
}
if detailed.Chunks[0].StartSegmentID != 1 {
t.Errorf("expected chunk[0].start_segment_id=1, got %d", detailed.Chunks[0].StartSegmentID)
}
if detailed.Chunks[0].EndSegmentID != 2 {
t.Errorf("expected chunk[0].end_segment_id=2, got %d", detailed.Chunks[0].EndSegmentID)
}
if detailed.Chunks[0].EstimatedTokens != 30 {
t.Errorf("expected chunk[0].estimated_tokens=30, got %d", detailed.Chunks[0].EstimatedTokens)
}
if detailed.Chunks[0].SegmentCount != 2 {
t.Errorf("expected chunk[0].segment_count=2, got %d", detailed.Chunks[0].SegmentCount)
}
// Check second chunk
if detailed.Chunks[1].Index != 1 {
t.Errorf("expected chunk[1].index=1, got %d", detailed.Chunks[1].Index)
}
if detailed.Chunks[1].StartSegmentID != 3 {
t.Errorf("expected chunk[1].start_segment_id=3, got %d", detailed.Chunks[1].StartSegmentID)
}
if detailed.Chunks[1].EndSegmentID != 5 {
t.Errorf("expected chunk[1].end_segment_id=5, got %d", detailed.Chunks[1].EndSegmentID)
}
if detailed.Chunks[1].SegmentCount != 3 {
t.Errorf("expected chunk[1].segment_count=3, got %d", detailed.Chunks[1].SegmentCount)
}
}

View File

@@ -1,7 +1,6 @@
package chunking
import (
"strings"
"unicode"
)
@@ -24,24 +23,24 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
words, punctuation := e.CountComponents(text)
return words + punctuation/2
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
// CountComponents returns unrounded counts for budgets composed from fragments.
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
inWord := false
for _, r := range text {
nonspace := !unicode.IsSpace(r)
if nonspace && !inWord {
words++
}
inWord = nonspace
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
punctuation++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
return words, punctuation
}
// ConstTokenEstimator returns a constant token count for testing purposes.

Some files were not shown because too many files have changed in this diff Show More