Run ordered processing with independent settings
This commit is contained in:
@@ -509,6 +509,12 @@ Use controlled candidates for both permitted edits and still-protected cases.
|
||||
|
||||
## Stage 12: Execute the ordered effective stages and extend preflight
|
||||
|
||||
**Completed.** The resolved pipeline supplies independent settings, vocabulary,
|
||||
and role clients to sequential execution; readiness and output checks precede
|
||||
work. Controlled processing covers carried-forward edits, distinct models,
|
||||
recoverable failures, an empty pipeline, and a late empty glossary. Older tests
|
||||
and stage-level attribution and artifacts remain for the following assignments.
|
||||
|
||||
Own runner/contracts and the processing handoff. Connect Stages 4, 8, 10, and
|
||||
11 in `internal/framework/runner`, `contracts`, `preflight`, and `internal/cli`.
|
||||
|
||||
|
||||
14
go.mod
14
go.mod
@@ -5,6 +5,12 @@ go 1.27.0
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||
gitea.maximumdirect.net/eric/promptkit v0.14.0
|
||||
go.maximumdirect.net/ckconfig v0.7.0
|
||||
go.maximumdirect.net/ckprompt v0.7.0
|
||||
go.maximumdirect.net/commandkit v0.6.0
|
||||
go.maximumdirect.net/configkit v1.2.0
|
||||
go.maximumdirect.net/promptkit v0.21.0
|
||||
go.maximumdirect.net/releasekit v0.9.0
|
||||
golang.org/x/mod v0.40.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
@@ -15,13 +21,9 @@ require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/spf13/cobra v1.10.1 // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||
go.maximumdirect.net/ckconfig v0.7.0
|
||||
go.maximumdirect.net/ckprompt v0.7.0
|
||||
go.maximumdirect.net/commandkit v0.6.0
|
||||
go.maximumdirect.net/configkit v1.2.0
|
||||
go.maximumdirect.net/promptkit v0.21.0
|
||||
go.maximumdirect.net/releasekit v0.9.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
|
||||
2
go.sum
2
go.sum
@@ -8,6 +8,7 @@ gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lz
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
@@ -27,6 +28,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
|
||||
@@ -110,7 +110,16 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
||||
}
|
||||
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||
}
|
||||
prepared, err := preflight.Prepare(ctx, preflight.Input{Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
||||
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
|
||||
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if profile.Backend != nil {
|
||||
_, err = loaded.Runtime.Engine.InspectBackend(ctx, *profile.Backend)
|
||||
}
|
||||
return err
|
||||
}, Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
166
internal/cli/process_ordered_integration_test.go
Normal file
166
internal/cli/process_ordered_integration_test.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
type controlledTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var models []string
|
||||
var prompts []string
|
||||
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var contents string
|
||||
for _, message := range request.Messages {
|
||||
contents += message.Content
|
||||
}
|
||||
mu.Lock()
|
||||
models = append(models, request.Model)
|
||||
prompts = append(prompts, contents)
|
||||
mu.Unlock()
|
||||
result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}`
|
||||
switch request.Model {
|
||||
case "model-one":
|
||||
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}`
|
||||
case "model-two":
|
||||
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}`
|
||||
case "model-three":
|
||||
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast","confidence":1}]}`
|
||||
}
|
||||
encoded, _ := json.Marshal(result)
|
||||
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": `version: 3
|
||||
reporting: {enabled: false}
|
||||
glossary_collections:
|
||||
party:
|
||||
entries: [{id: alice, name: Alice, category: person}]
|
||||
spells:
|
||||
entries: [{id: shield, name: Shield, category: spell}]
|
||||
protected_terms: [party, spells]
|
||||
llm: {validation_profile: review}
|
||||
pipeline:
|
||||
stages:
|
||||
- module: glossary
|
||||
glossaries: [party]
|
||||
llm: {proposal_profile: one}
|
||||
- module: glossary
|
||||
glossaries: [spells]
|
||||
llm: {proposal_profile: two}
|
||||
- module: grammar
|
||||
glossaries: [party, spells]
|
||||
llm: {proposal_profile: three}
|
||||
`,
|
||||
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
|
||||
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
|
||||
}
|
||||
for name, value := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
|
||||
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
|
||||
}
|
||||
if len(prompts) > 3 && !strings.Contains(prompts[3], "Alice casts Sheeld.") {
|
||||
t.Fatalf("second proposal missed earlier edit: %s", prompts[3])
|
||||
}
|
||||
if !reflect.DeepEqual(models, []string{"model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||
t.Fatalf("models=%v", models)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
files := map[string]string{
|
||||
"config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar}]\n",
|
||||
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
|
||||
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||
}
|
||||
for name, value := range files {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
|
||||
})
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer prepared.Prepared.Output.Close()
|
||||
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" {
|
||||
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
|
||||
}
|
||||
}
|
||||
53
internal/cli/process_pipeline.go
Normal file
53
internal/cli/process_pipeline.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
)
|
||||
|
||||
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
|
||||
cfg := &loaded.Configuration.Config
|
||||
specs := contracts.ResolveModuleRunSpecs(cfg)
|
||||
observer := llm.NewObservationCollector()
|
||||
for i := range specs {
|
||||
item := &specs[i]
|
||||
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, observer.Hooks())
|
||||
if err != nil {
|
||||
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
|
||||
}
|
||||
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, observer.Hooks())
|
||||
if err != nil {
|
||||
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
|
||||
}
|
||||
item.ProposalClient, item.ValidationClient = proposal, validation
|
||||
}
|
||||
debugPath := prepared.DebugPath()
|
||||
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
|
||||
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
|
||||
})
|
||||
output.Observations = observer.Snapshot()
|
||||
output.ObservationsAvailable = true
|
||||
output.LLMSummary = llm.Summarize(output.Observations)
|
||||
if debug := prepared.Prepared.Debug; debug != nil {
|
||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations)))
|
||||
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||
}
|
||||
var failures []processreport.RunFailure
|
||||
if runErr != nil {
|
||||
failures = append(failures, processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure})
|
||||
}
|
||||
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, ModuleSpecs: specs, RunOutput: &output, RunFailures: failures})
|
||||
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
||||
}
|
||||
@@ -24,8 +24,8 @@ type LLMSummary struct {
|
||||
}
|
||||
|
||||
type BackendUtilization struct {
|
||||
Concurrency int `json:"concurrency"`
|
||||
QueueCapacity int `json:"queue_capacity"`
|
||||
Concurrency *int `json:"concurrency,omitempty"`
|
||||
QueueCapacity *int `json:"queue_capacity,omitempty"`
|
||||
LLMSummary
|
||||
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
||||
GenerationMS int64 `json:"generation_ms"`
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type GlossaryEntry struct {
|
||||
Name string `yaml:"name"`
|
||||
Aliases []string `yaml:"aliases,omitempty"`
|
||||
Plural string `yaml:"plural,omitempty"`
|
||||
Category string `yaml:"category"`
|
||||
Summary string `yaml:"summary"`
|
||||
}
|
||||
|
||||
type Glossary struct {
|
||||
Entries []GlossaryEntry `yaml:"glossary"`
|
||||
}
|
||||
|
||||
func ParseGlossaryYAML(raw []byte) (*Glossary, error) {
|
||||
var top map[string]any
|
||||
if err := yaml.Unmarshal(raw, &top); err != nil {
|
||||
return nil, &ParseError{Message: fmt.Sprintf("glossary is not valid YAML: %v", err)}
|
||||
}
|
||||
|
||||
var g Glossary
|
||||
if err := yaml.Unmarshal(raw, &g); err != nil {
|
||||
return nil, &ParseError{Message: fmt.Sprintf("failed to parse glossary: %v", err)}
|
||||
}
|
||||
|
||||
if err := validateGlossary(&g); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &g, nil
|
||||
}
|
||||
|
||||
func validateGlossary(g *Glossary) error {
|
||||
if len(g.Entries) == 0 {
|
||||
return &ValidationError{Field: "glossary", Message: "must contain at least one entry"}
|
||||
}
|
||||
|
||||
for i, e := range g.Entries {
|
||||
entryLabel := fmt.Sprintf("glossary[%d]", i)
|
||||
|
||||
if e.Name == "" {
|
||||
return &ValidationError{Field: fmt.Sprintf("%s.name", entryLabel), Message: "must not be empty"}
|
||||
}
|
||||
|
||||
if e.Category == "" {
|
||||
return &ValidationError{Field: fmt.Sprintf("%s.category", entryLabel), Message: "must not be empty"}
|
||||
}
|
||||
|
||||
if e.Summary == "" {
|
||||
return &ValidationError{Field: fmt.Sprintf("%s.summary", entryLabel), Message: "must not be empty"}
|
||||
}
|
||||
|
||||
for j, alias := range e.Aliases {
|
||||
if alias == "" {
|
||||
return &ValidationError{
|
||||
Field: fmt.Sprintf("%s.aliases[%d]", entryLabel, j),
|
||||
Message: "must not be empty",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if e.Plural != "" && len(e.Plural) == 0 {
|
||||
return &ValidationError{Field: fmt.Sprintf("%s.plural", entryLabel), Message: "must not be empty if present"}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseGlossaryYAML_Valid(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/glossary_valid.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
glossary, err := ParseGlossaryYAML(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseGlossaryYAML failed: %v", err)
|
||||
}
|
||||
|
||||
if len(glossary.Entries) != 2 {
|
||||
t.Fatalf("expected 2 entries, got %d", len(glossary.Entries))
|
||||
}
|
||||
|
||||
e1 := glossary.Entries[0]
|
||||
if e1.Name != "Jesters" {
|
||||
t.Errorf("expected entries[0].name = Jesters, got %q", e1.Name)
|
||||
}
|
||||
if e1.Category != "faction" {
|
||||
t.Errorf("expected entries[0].category = faction, got %q", e1.Category)
|
||||
}
|
||||
if e1.Summary != "A faction name." {
|
||||
t.Errorf("expected entries[0].summary = A faction name., got %q", e1.Summary)
|
||||
}
|
||||
if len(e1.Aliases) != 0 {
|
||||
t.Errorf("expected entries[0].aliases = [], got %v", e1.Aliases)
|
||||
}
|
||||
|
||||
e2 := glossary.Entries[1]
|
||||
if e2.Name != "Popov" {
|
||||
t.Errorf("expected entries[1].name = Popov, got %q", e2.Name)
|
||||
}
|
||||
if len(e2.Aliases) != 1 || e2.Aliases[0] != "Hrank" {
|
||||
t.Errorf("expected entries[1].aliases = [Hrank], got %v", e2.Aliases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGlossaryYAML_MalformedYAML(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/glossary_malformed.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
_, err = ParseGlossaryYAML(raw)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for malformed YAML, got nil")
|
||||
}
|
||||
|
||||
parseErr, ok := err.(*ParseError)
|
||||
if !ok {
|
||||
t.Errorf("expected *ParseError, got %T", err)
|
||||
} else if parseErr.Message == "" {
|
||||
t.Error("expected non-empty parse error message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGlossaryYAML_MissingRequiredFields(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/glossary_missing_fields.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read test fixture: %v", err)
|
||||
}
|
||||
|
||||
_, err = ParseGlossaryYAML(raw)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing required fields, got nil")
|
||||
}
|
||||
|
||||
valErr, ok := err.(*ValidationError)
|
||||
if !ok {
|
||||
t.Errorf("expected *ValidationError, got %T", err)
|
||||
} else if valErr.Field == "" {
|
||||
t.Error("expected non-empty validation error field")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGlossaryYAML_EmptyGlossary(t *testing.T) {
|
||||
raw := []byte(`glossary: []`)
|
||||
|
||||
_, err := ParseGlossaryYAML(raw)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty glossary, got nil")
|
||||
}
|
||||
|
||||
valErr, ok := err.(*ValidationError)
|
||||
if !ok {
|
||||
t.Errorf("expected *ValidationError, got %T", err)
|
||||
} else if valErr.Message != "must contain at least one entry" {
|
||||
t.Errorf("expected 'must contain at least one entry', got %q", valErr.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGlossaryYAML_EmptyName(t *testing.T) {
|
||||
raw := []byte(`
|
||||
glossary:
|
||||
- name: ""
|
||||
category: faction
|
||||
summary: A faction.
|
||||
`)
|
||||
|
||||
_, err := ParseGlossaryYAML(raw)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty name, got nil")
|
||||
}
|
||||
|
||||
valErr, ok := err.(*ValidationError)
|
||||
if !ok {
|
||||
t.Errorf("expected *ValidationError, got %T", err)
|
||||
} else if valErr.Message != "must not be empty" {
|
||||
t.Errorf("expected 'must not be empty', got %q", valErr.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGlossaryYAML_EmptyAlias(t *testing.T) {
|
||||
raw := []byte(`
|
||||
glossary:
|
||||
- name: Test
|
||||
aliases:
|
||||
- ""
|
||||
category: faction
|
||||
summary: A test entry.
|
||||
`)
|
||||
|
||||
_, err := ParseGlossaryYAML(raw)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty alias, got nil")
|
||||
}
|
||||
|
||||
valErr, ok := err.(*ValidationError)
|
||||
if !ok {
|
||||
t.Errorf("expected *ValidationError, got %T", err)
|
||||
} else if valErr.Message != "must not be empty" {
|
||||
t.Errorf("expected 'must not be empty', got %q", valErr.Message)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -38,6 +39,13 @@ type ModuleRunSpec struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
InstanceName string `json:"instance_name"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Index int `json:"-"`
|
||||
ID *string `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
ProposalClient StructuredLLMClient `json:"-"`
|
||||
ValidationClient StructuredLLMClient `json:"-"`
|
||||
}
|
||||
|
||||
// SectionMetadata captures chunk-level metadata without embedding full segment payloads.
|
||||
@@ -64,7 +72,7 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
|
||||
type ExecutionContext struct {
|
||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
@@ -87,32 +95,20 @@ type ProposalResult struct {
|
||||
// ValidationRequest is the input to validator execution.
|
||||
type ValidationRequest = validators.Request
|
||||
|
||||
// ResolveModuleRunSpecs deterministically resolves instance names from logical keys.
|
||||
// Repeated keys are suffixed with _<n> (1-based), while singleton keys keep their raw key.
|
||||
func ResolveModuleRunSpecs(moduleKeys []string) ([]ModuleRunSpec, error) {
|
||||
totals := make(map[string]int, len(moduleKeys))
|
||||
for i, key := range moduleKeys {
|
||||
if key == "" {
|
||||
return nil, fmt.Errorf("module key at index %d must not be empty", i)
|
||||
}
|
||||
totals[key]++
|
||||
// ResolveModuleRunSpecs derives isolated consumer views from resolved settings.
|
||||
func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
seen := make(map[string]int, len(totals))
|
||||
specs := make([]ModuleRunSpec, len(moduleKeys))
|
||||
for i, key := range moduleKeys {
|
||||
seen[key]++
|
||||
|
||||
name := key
|
||||
if totals[key] > 1 {
|
||||
name = fmt.Sprintf("%s_%d", key, seen[key])
|
||||
}
|
||||
|
||||
specs[i] = ModuleRunSpec{
|
||||
ModuleKey: key,
|
||||
InstanceName: name,
|
||||
specs := make([]ModuleRunSpec, len(cfg.Pipeline.Stages))
|
||||
for i, item := range cfg.Pipeline.Stages {
|
||||
selected := glossary.Select(cfg.GlossaryCollections, item.Glossaries)
|
||||
protected := glossary.Select(cfg.GlossaryCollections, item.ProtectedTerms)
|
||||
var excluded []glossary.Entry
|
||||
if item.Module == modulecatalog.KeyGlossary {
|
||||
excluded = selected
|
||||
}
|
||||
specs[i] = ModuleRunSpec{ModuleKey: item.Module, InstanceName: fmt.Sprintf("%s_%d", item.Module, i+1), Index: i, ID: item.ID, Settings: item, Glossary: selected, ProtectedTerms: glossary.Vocabulary(protected, excluded)}
|
||||
}
|
||||
|
||||
return specs, nil
|
||||
return specs
|
||||
}
|
||||
|
||||
@@ -5,9 +5,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
glossarymodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
|
||||
grammarmodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||
@@ -32,19 +30,9 @@ func IsKnownModuleKey(key string) bool {
|
||||
return modulecatalog.IsSupported(key)
|
||||
}
|
||||
|
||||
// Dependencies holds explicit constructor dependencies for module creation.
|
||||
type Dependencies struct {
|
||||
Config *config.Config
|
||||
Glossary *schema.Glossary
|
||||
ProposalLLMClient contracts.StructuredLLMClient
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
}
|
||||
|
||||
// ConstructRequest is one module-construction request.
|
||||
// ConstructRequest identifies the module to construct.
|
||||
type ConstructRequest struct {
|
||||
RunSpec contracts.ModuleRunSpec
|
||||
Dependencies
|
||||
}
|
||||
|
||||
// Constructor builds one module instance from a run spec and explicit deps.
|
||||
@@ -52,14 +40,12 @@ type Constructor func(ctx context.Context, req ConstructRequest) (contracts.Tran
|
||||
|
||||
// Factory resolves configured module specs into module instances.
|
||||
type Factory struct {
|
||||
deps Dependencies
|
||||
constructors map[string]Constructor
|
||||
}
|
||||
|
||||
// NewFactory creates a production module registry.
|
||||
func NewFactory(deps Dependencies) *Factory {
|
||||
func NewFactory() *Factory {
|
||||
factory := &Factory{
|
||||
deps: deps,
|
||||
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
|
||||
}
|
||||
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
||||
@@ -125,16 +111,7 @@ func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Transcr
|
||||
return nil, fmt.Errorf("internal module registry error: constructor for module %q is not configured", key)
|
||||
}
|
||||
|
||||
module, err := constructor(context.Background(), ConstructRequest{
|
||||
RunSpec: spec,
|
||||
Dependencies: Dependencies{
|
||||
Config: f.deps.Config,
|
||||
Glossary: f.deps.Glossary,
|
||||
ProposalLLMClient: f.deps.ProposalLLMClient,
|
||||
ValidationLLMClient: f.deps.ValidationLLMClient,
|
||||
DiagnosticsDir: f.deps.DiagnosticsDir,
|
||||
},
|
||||
})
|
||||
module, err := constructor(context.Background(), ConstructRequest{RunSpec: spec})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct module %q: %w", spec.InstanceName, err)
|
||||
}
|
||||
|
||||
@@ -7,11 +7,14 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
// Input contains already acquired application and transcript inputs.
|
||||
type Input struct {
|
||||
InspectProfile func(context.Context, string) error
|
||||
Config *config.Config
|
||||
Transcript *schema.SourceTranscript
|
||||
Output diagnostics.OutputSelection
|
||||
@@ -45,6 +48,9 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
|
||||
if input.Transcript == nil {
|
||||
return nil, fmt.Errorf("transcript is required")
|
||||
}
|
||||
if err := CheckReadiness(ctx, input.Config, input.InspectProfile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
session, err := diagnostics.SelectOutput(input.Output)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -74,3 +80,38 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// CheckReadiness checks all resolved processing selections before output preparation.
|
||||
func CheckReadiness(ctx context.Context, cfg *config.Config, inspectProfile func(context.Context, string) error) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("application configuration is required")
|
||||
}
|
||||
label := func(index int, item config.Stage) string {
|
||||
if item.ID != nil {
|
||||
return fmt.Sprintf("stage %d (%s, id %q)", index+1, item.Module, *item.ID)
|
||||
}
|
||||
return fmt.Sprintf("stage %d (%s)", index+1, item.Module)
|
||||
}
|
||||
for index, item := range cfg.Pipeline.Stages {
|
||||
if item.Module == modulecatalog.KeyGlossary && len(glossary.Select(cfg.GlossaryCollections, item.Glossaries)) == 0 {
|
||||
return fmt.Errorf("%s has no glossary entries in selected collections %q", label(index, item), item.Glossaries)
|
||||
}
|
||||
}
|
||||
for index, item := range cfg.Pipeline.Stages {
|
||||
for _, selection := range []struct {
|
||||
role string
|
||||
profile *string
|
||||
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}} {
|
||||
if selection.profile == nil || *selection.profile == "" {
|
||||
return fmt.Errorf("%s requires a %s profile", label(index, item), selection.role)
|
||||
}
|
||||
if inspectProfile == nil {
|
||||
return fmt.Errorf("profile inspection is required")
|
||||
}
|
||||
if err := inspectProfile(ctx, *selection.profile); err != nil {
|
||||
return fmt.Errorf("%s %s profile %q: %w", label(index, item), selection.role, *selection.profile, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
39
internal/framework/preflight/readiness_test.go
Normal file
39
internal/framework/preflight/readiness_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestLaterEmptyGlossaryFailsBeforeOutputPreparation(t *testing.T) {
|
||||
profile, id := "selected", "spells"
|
||||
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{
|
||||
{Module: "grammar", LLM: config.LLM{ProposalProfile: &profile, ValidationProfile: &profile}},
|
||||
{Module: "glossary", ID: &id, Glossaries: []string{"empty"}, LLM: config.LLM{ProposalProfile: &profile, ValidationProfile: &profile}},
|
||||
}}}
|
||||
output := filepath.Join(t.TempDir(), "transcript.json")
|
||||
calls := 0
|
||||
_, err := Prepare(context.Background(), Input{Config: cfg, Transcript: &schema.SourceTranscript{}, Output: diagnostics.OutputSelection{TranscriptPath: output}, InspectProfile: func(context.Context, string) error { calls++; return nil }})
|
||||
if err == nil || !strings.Contains(err.Error(), "stage 2") || !strings.Contains(err.Error(), "spells") || !strings.Contains(err.Error(), "empty") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Fatal("inspected profiles before checking all glossary selections")
|
||||
}
|
||||
if _, err := os.Stat(output); !os.IsNotExist(err) {
|
||||
t.Fatalf("output exists: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyPipelineNeedsNoProfile(t *testing.T) {
|
||||
if err := CheckReadiness(context.Background(), &config.Config{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,11 @@ package processreport
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
@@ -23,7 +22,7 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
}
|
||||
|
||||
// BuildUtilization groups complete logical calls by their selected backend.
|
||||
func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backend) reporting.UtilizationDiagnostics {
|
||||
func BuildUtilization(output *runner.RunOutput, backends map[string]*pk.BackendInspection) reporting.UtilizationDiagnostics {
|
||||
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
||||
if output == nil {
|
||||
return result
|
||||
@@ -85,7 +84,7 @@ func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backe
|
||||
}
|
||||
}
|
||||
result.Backends[id] = reporting.BackendUtilization{
|
||||
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
||||
Concurrency: backend.ConcurrencyLimit, QueueCapacity: backend.QueueCapacity,
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
|
||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
@@ -57,7 +56,6 @@ type Request struct {
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
Config *config.Config `json:"-"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
|
||||
@@ -35,10 +35,7 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
sectionIndex = req.ProposalRequest.Section.Index
|
||||
}
|
||||
|
||||
transcriptDescription := ""
|
||||
if req.ProposalRequest.Config != nil {
|
||||
transcriptDescription = req.ProposalRequest.Config.Context.Description
|
||||
}
|
||||
transcriptDescription := req.ProposalRequest.Settings.Context.Description
|
||||
|
||||
variables, err := req.BuildVariables(
|
||||
req.ProposalRequest.WorkingTranscript,
|
||||
@@ -61,7 +58,6 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||
Section: req.ProposalRequest.Section,
|
||||
Config: req.ProposalRequest.Config,
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
|
||||
67
internal/framework/runner/ordered_workflow_test.go
Normal file
67
internal/framework/runner/ordered_workflow_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type orderedFactory struct {
|
||||
seen chan contracts.ProposalRequest
|
||||
}
|
||||
|
||||
func (f orderedFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
|
||||
return orderedModule{spec: spec, seen: f.seen}, nil
|
||||
}
|
||||
|
||||
type orderedModule struct {
|
||||
spec contracts.ModuleRunSpec
|
||||
seen chan contracts.ProposalRequest
|
||||
}
|
||||
|
||||
func (m orderedModule) Key() string { return m.spec.ModuleKey }
|
||||
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m orderedModule) Validators() []contracts.Validator { return nil }
|
||||
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
m.seen <- req
|
||||
original, corrected := "party", "Party"
|
||||
if m.spec.Index == 1 {
|
||||
original, corrected = "spell", "Spell"
|
||||
}
|
||||
if m.spec.Index == 2 {
|
||||
original, corrected = "Party Spell", "Party Spell!"
|
||||
}
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected, Confidence: 1}}}, nil
|
||||
}
|
||||
|
||||
func TestOrderedWorkflowUsesIndependentSelectionsAndPriorEdits(t *testing.T) {
|
||||
cfg := &config.Config{GlossaryCollections: map[string]config.GlossaryCollection{
|
||||
"party": {Entries: []config.GlossaryEntry{{ID: "party", Name: "Party", Category: "person"}}},
|
||||
"spells": {Entries: []config.GlossaryEntry{{ID: "spell", Name: "Spell", Category: "spell"}}},
|
||||
}, Pipeline: config.Pipeline{Stages: []config.Stage{
|
||||
{Module: "glossary", Glossaries: []string{"party"}, ProtectedTerms: []string{"party", "spells"}, Context: config.Context{Description: "party context"}},
|
||||
{Module: "glossary", Glossaries: []string{"spells"}, ProtectedTerms: []string{"party", "spells"}, Context: config.Context{Description: "spell context"}},
|
||||
{Module: "grammar", Glossaries: []string{"party", "spells"}, ProtectedTerms: []string{"party", "spells"}},
|
||||
}}}
|
||||
seen := make(chan contracts.ProposalRequest, 3)
|
||||
output, err := New(orderedFactory{seen: seen}).Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "party spell"}}}, ModuleSpecs: contracts.ResolveModuleRunSpecs(cfg)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := output.FinalTranscript.Segments[0].Text; got != "Party Spell!" {
|
||||
t.Fatalf("final text = %q", got)
|
||||
}
|
||||
first, second, third := <-seen, <-seen, <-seen
|
||||
if first.Settings.Context.Description != "party context" || second.Settings.Context.Description != "spell context" || len(first.Glossary) != 1 || first.Glossary[0].Name != "Party" || len(second.Glossary) != 1 || second.Glossary[0].Name != "Spell" || len(third.Glossary) != 2 {
|
||||
t.Fatalf("incorrect context: %+v %+v %+v", first, second, third)
|
||||
}
|
||||
if second.WorkingTranscript.Segments[0].Text != "Party spell" || third.WorkingTranscript.Segments[0].Text != "Party Spell" {
|
||||
t.Fatalf("edits not carried forward: %q, %q", second.WorkingTranscript.Segments[0].Text, third.WorkingTranscript.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
@@ -93,11 +94,8 @@ type RunInput struct {
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Config *config.Config
|
||||
Transcript *schema.Transcript
|
||||
Glossary *schema.Glossary
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
ProposalLLMClient contracts.StructuredLLMClient
|
||||
ProposalDiagnosticsDir string
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
ValidationDiagnosticsDir string
|
||||
CaptureCandidates bool
|
||||
timings *validatorTimingCollector
|
||||
@@ -185,12 +183,10 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
Module: module,
|
||||
Spec: spec,
|
||||
Policy: policy,
|
||||
Config: input.Config,
|
||||
Glossary: input.Glossary,
|
||||
Sections: sections,
|
||||
ProposalClient: input.ProposalLLMClient,
|
||||
ProposalClient: spec.ProposalClient,
|
||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||
ValidationClient: input.ValidationLLMClient,
|
||||
ValidationClient: spec.ValidationClient,
|
||||
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||
Timings: input.timings,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
@@ -220,7 +216,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
|
||||
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
|
||||
}
|
||||
|
||||
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy)
|
||||
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, validators.NewProtectedVocabulary(spec.ProtectedTerms), spec.ModuleKey == modulecatalog.KeySpokenWord)
|
||||
if applyErr != nil {
|
||||
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
|
||||
}
|
||||
@@ -268,7 +264,7 @@ func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
|
||||
|
||||
// applyWithinSections leaves section membership and initial metadata unchanged.
|
||||
// Approved proposals are applied only to their identified section.
|
||||
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
||||
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary validators.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
||||
bySection := make(map[int][]proposals.EnrichedCorrectionProposal, len(sections))
|
||||
known := make(map[int]struct{}, len(sections))
|
||||
for _, section := range sections {
|
||||
@@ -289,7 +285,7 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
||||
skipped := make([]proposals.SkippedChange, 0)
|
||||
appliedBySection := make([]int, len(sections))
|
||||
for i, section := range sections {
|
||||
result := proposals.ApplyProposals(transcriptFromSection(section), bySection[section.Index], policy)
|
||||
result := proposals.ApplyProposals(transcriptFromSection(section), bySection[section.Index], policy, func(before, after string) bool { return vocabulary.AllowsChange(before, after, allowRepetitionCleanup) })
|
||||
updated[i].Segments = result.Transcript.Segments
|
||||
applied = append(applied, result.Applied...)
|
||||
skipped = append(skipped, result.Skipped...)
|
||||
@@ -304,8 +300,6 @@ type collectSectionProposalsInput struct {
|
||||
Module contracts.TranscriptModule
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Config *config.Config
|
||||
Glossary *schema.Glossary
|
||||
Sections []chunking.Section
|
||||
ProposalClient contracts.StructuredLLMClient
|
||||
DiagnosticsDir string
|
||||
@@ -364,9 +358,10 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
meta := contracts.SectionMetadataFromSection(section)
|
||||
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{
|
||||
Config: input.Config,
|
||||
Settings: input.Spec.Settings,
|
||||
WorkingTranscript: transcriptFromSection(section),
|
||||
Glossary: input.Glossary,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
Section: &meta,
|
||||
DiagnosticsDir: input.DiagnosticsDir,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
@@ -506,8 +501,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
|
||||
Spec: input.Spec,
|
||||
Policy: input.Policy,
|
||||
Glossary: input.Glossary,
|
||||
Config: input.Config,
|
||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
@@ -592,8 +585,6 @@ type validateSectionCandidatesInput struct {
|
||||
ReportDiagnosticError func(error) `json:"-"`
|
||||
Spec contracts.ModuleRunSpec
|
||||
Policy proposals.ReplacementPolicy
|
||||
Glossary *schema.Glossary
|
||||
Config *config.Config
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
@@ -624,10 +615,16 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}
|
||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||
if input.DiagnosticsDir != "" {
|
||||
var secrets []string
|
||||
if source, ok := input.ValidationLLMClient.(interface {
|
||||
ConfiguredSecrets(context.Context) []string
|
||||
}); ok {
|
||||
secrets = source.ConfiguredSecrets(ctx)
|
||||
}
|
||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||
llm.ConfiguredSecrets(input.Config),
|
||||
secrets,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -641,8 +638,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
SectionIndex: input.SectionIndex,
|
||||
ValidatorOrder: &validatorOrder,
|
||||
ReplacementPolicy: input.Policy,
|
||||
Glossary: input.Glossary,
|
||||
Config: input.Config,
|
||||
Glossary: input.Spec.Glossary,
|
||||
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||
Settings: input.Spec.Settings,
|
||||
LLMClient: input.ValidationLLMClient,
|
||||
DiagnosticsWriter: diagnosticsWriter,
|
||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||
|
||||
@@ -39,7 +39,7 @@ type ConfidenceThresholdValidator struct{}
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
|
||||
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
threshold := confidenceThresholdForModule(req.ModuleKey, req.Config)
|
||||
threshold := req.Settings.ConfidenceThreshold
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if c.Confidence < threshold {
|
||||
|
||||
@@ -72,8 +72,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
}
|
||||
|
||||
maxTokens := config.DefaultValidationMaxPromptTokens
|
||||
if req.Config != nil && req.Config.Validation.MaxPromptTokens > 0 {
|
||||
maxTokens = req.Config.Validation.MaxPromptTokens
|
||||
if req.Settings.Validation.MaxPromptTokens > 0 {
|
||||
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||
}
|
||||
|
||||
for i := range validationReq.Items {
|
||||
@@ -131,9 +131,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
return partialResult(), err
|
||||
}
|
||||
transcriptDescription := ""
|
||||
if req.Config != nil {
|
||||
transcriptDescription = req.Config.Context.Description
|
||||
}
|
||||
transcriptDescription = req.Settings.Context.Description
|
||||
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
|
||||
if err != nil {
|
||||
return partialResult(), err
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
@@ -44,7 +43,7 @@ type Request struct {
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
Glossary []glossary.Entry `json:"-"`
|
||||
ProtectedTerms []glossary.Term `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
Settings config.Stage `json:"-"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
||||
}
|
||||
@@ -142,21 +141,3 @@ func rejection(index int, reasonCode string, msg string) Decision {
|
||||
func unavailable(index int, reasonCode string, msg string) Decision {
|
||||
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionUnavailable, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
|
||||
}
|
||||
|
||||
func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {
|
||||
if cfg == nil {
|
||||
return 0.0
|
||||
}
|
||||
switch moduleKey {
|
||||
case modulecatalog.KeyGlossary:
|
||||
return cfg.Thresholds.Glossary
|
||||
case modulecatalog.KeyGrammar:
|
||||
return cfg.Thresholds.Grammar
|
||||
case modulecatalog.KeyHomophones:
|
||||
return cfg.Thresholds.Homophones
|
||||
case modulecatalog.KeySpokenWord:
|
||||
return cfg.Thresholds.SpokenWord
|
||||
default:
|
||||
return 0.0
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user