Run ordered processing with independent settings
This commit is contained in:
@@ -509,6 +509,12 @@ Use controlled candidates for both permitted edits and still-protected cases.
|
|||||||
|
|
||||||
## Stage 12: Execute the ordered effective stages and extend preflight
|
## Stage 12: Execute the ordered effective stages and extend preflight
|
||||||
|
|
||||||
|
**Completed.** The resolved pipeline supplies independent settings, vocabulary,
|
||||||
|
and role clients to sequential execution; readiness and output checks precede
|
||||||
|
work. Controlled processing covers carried-forward edits, distinct models,
|
||||||
|
recoverable failures, an empty pipeline, and a late empty glossary. Older tests
|
||||||
|
and stage-level attribution and artifacts remain for the following assignments.
|
||||||
|
|
||||||
Own runner/contracts and the processing handoff. Connect Stages 4, 8, 10, and
|
Own runner/contracts and the processing handoff. Connect Stages 4, 8, 10, and
|
||||||
11 in `internal/framework/runner`, `contracts`, `preflight`, and `internal/cli`.
|
11 in `internal/framework/runner`, `contracts`, `preflight`, and `internal/cli`.
|
||||||
|
|
||||||
|
|||||||
14
go.mod
14
go.mod
@@ -5,6 +5,12 @@ go 1.27.0
|
|||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||||
gitea.maximumdirect.net/eric/promptkit v0.14.0
|
gitea.maximumdirect.net/eric/promptkit v0.14.0
|
||||||
|
go.maximumdirect.net/ckconfig v0.7.0
|
||||||
|
go.maximumdirect.net/ckprompt v0.7.0
|
||||||
|
go.maximumdirect.net/commandkit v0.6.0
|
||||||
|
go.maximumdirect.net/configkit v1.2.0
|
||||||
|
go.maximumdirect.net/promptkit v0.21.0
|
||||||
|
go.maximumdirect.net/releasekit v0.9.0
|
||||||
golang.org/x/mod v0.40.0
|
golang.org/x/mod v0.40.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
@@ -15,13 +21,9 @@ require (
|
|||||||
github.com/kr/pretty v0.3.1 // indirect
|
github.com/kr/pretty v0.3.1 // indirect
|
||||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||||
|
github.com/spf13/cobra v1.10.1 // indirect
|
||||||
|
github.com/spf13/pflag v1.0.9 // indirect
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.maximumdirect.net/ckconfig v0.7.0
|
|
||||||
go.maximumdirect.net/ckprompt v0.7.0
|
|
||||||
go.maximumdirect.net/commandkit v0.6.0
|
|
||||||
go.maximumdirect.net/configkit v1.2.0
|
|
||||||
go.maximumdirect.net/promptkit v0.21.0
|
|
||||||
go.maximumdirect.net/releasekit v0.9.0
|
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -8,6 +8,7 @@ gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lz
|
|||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
|
github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
@@ -27,6 +28,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
|
|||||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||||
|
|||||||
@@ -110,7 +110,16 @@ func prepareProcess(ctx context.Context, input processInput, loaded *processInpu
|
|||||||
}
|
}
|
||||||
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
debug = &preflight.DebugInput{WorkDir: cfg.Diagnostics.WorkDir, RunID: runID, Secrets: secrets, EffectiveConfig: effective.Bytes(), Metadata: diagnostics.InvocationMetadata{Operation: "process", TranscriptPath: input.Transcript, OutputPath: input.Output.Value, Overlay: input.Overlay.Value, ConfigVersion: &cfg.Version, RunID: runID, StartedAt: started}}
|
||||||
}
|
}
|
||||||
prepared, err := preflight.Prepare(ctx, preflight.Input{Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
prepared, err := preflight.Prepare(ctx, preflight.Input{InspectProfile: func(ctx context.Context, id string) error {
|
||||||
|
profile, err := loaded.Runtime.Engine.InspectProfile(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if profile.Backend != nil {
|
||||||
|
_, err = loaded.Runtime.Engine.InspectBackend(ctx, *profile.Backend)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}, Config: cfg, Transcript: loaded.Source, Output: diagnostics.OutputSelection{InputTranscriptPath: input.Transcript, TranscriptPath: input.Output.Value, ReportDir: reportDir, ReportingEnabled: cfg.Reporting.Enabled, ProtectedPaths: loaded.ProtectedPaths, Stdout: stdout}, InitialReports: initial, FailedReports: failed, Debug: debug})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
166
internal/cli/process_ordered_integration_test.go
Normal file
166
internal/cli/process_ordered_integration_test.go
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
ck "go.maximumdirect.net/configkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type controlledTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f controlledTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestProcessingUsesPerEntryProfilesInOrder(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var models []string
|
||||||
|
var prompts []string
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
var request struct {
|
||||||
|
Model string `json:"model"`
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var contents string
|
||||||
|
for _, message := range request.Messages {
|
||||||
|
contents += message.Content
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
models = append(models, request.Model)
|
||||||
|
prompts = append(prompts, contents)
|
||||||
|
mu.Unlock()
|
||||||
|
result := `{"validations":[{"correction_index":1,"approved":true,"confidence":1,"reason":"ok"}]}`
|
||||||
|
switch request.Model {
|
||||||
|
case "model-one":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Alyce","corrected_text":"Alice","confidence":1}]}`
|
||||||
|
case "model-two":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"Sheeld","corrected_text":"Shield","confidence":1}]}`
|
||||||
|
case "model-three":
|
||||||
|
result = `{"corrections":[{"id":1,"original_text":"casts","corrected_text":"cast","confidence":1}]}`
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(result)
|
||||||
|
body := fmt.Sprintf(`{"model":"fixture","choices":[{"message":{"role":"assistant","content":%s}}]}`, encoded)
|
||||||
|
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||||
|
})
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": `version: 3
|
||||||
|
reporting: {enabled: false}
|
||||||
|
glossary_collections:
|
||||||
|
party:
|
||||||
|
entries: [{id: alice, name: Alice, category: person}]
|
||||||
|
spells:
|
||||||
|
entries: [{id: shield, name: Shield, category: spell}]
|
||||||
|
protected_terms: [party, spells]
|
||||||
|
llm: {validation_profile: review}
|
||||||
|
pipeline:
|
||||||
|
stages:
|
||||||
|
- module: glossary
|
||||||
|
glossaries: [party]
|
||||||
|
llm: {proposal_profile: one}
|
||||||
|
- module: glossary
|
||||||
|
glossaries: [spells]
|
||||||
|
llm: {proposal_profile: two}
|
||||||
|
- module: grammar
|
||||||
|
glossaries: [party, spells]
|
||||||
|
llm: {proposal_profile: three}
|
||||||
|
`,
|
||||||
|
"prompts.yml": fmt.Sprintf("profiles:\n - {id: one, endpoint: %q, model: model-one, api_key_required: false}\n - {id: two, endpoint: %q, model: model-two, api_key_required: false}\n - {id: three, endpoint: %q, model: model-three, api_key_required: false}\n - {id: review, endpoint: %q, model: review, api_key_required: false}\n", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1", "https://example.invalid/v1"),
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"Alyce casts Sheeld."}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "Alice cast Shield." {
|
||||||
|
t.Fatalf("result=%q models=%v", result.Transcript.Segments[0].Text, models)
|
||||||
|
}
|
||||||
|
if len(prompts) > 3 && !strings.Contains(prompts[3], "Alice casts Sheeld.") {
|
||||||
|
t.Fatalf("second proposal missed earlier edit: %s", prompts[3])
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(models, []string{"model-one", "review", "review", "model-two", "review", "review", "model-three", "review", "review"}) {
|
||||||
|
t.Fatalf("models=%v", models)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecoverableModelFailureKeepsNormalizedTranscript(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
files := map[string]string{
|
||||||
|
"config.yml": "version: 3\nreporting: {enabled: false}\nllm: {proposal_profile: local, transient_retries: 0}\npipeline:\n stages: [{module: grammar}]\n",
|
||||||
|
"prompts.yml": "profiles:\n - {id: local, endpoint: 'https://example.invalid/v1', model: fixture, api_key_required: false}\n",
|
||||||
|
"transcript.json": `[{"speaker":"Alice","start":0,"end":1,"text":"hello"}]`,
|
||||||
|
}
|
||||||
|
for name, value := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
transport := controlledTransport(func(r *http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: http.StatusServiceUnavailable, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"error":{"message":"temporarily unavailable"}}`)), Request: r}, nil
|
||||||
|
})
|
||||||
|
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
source, err := schema.ParseSourceTranscriptJSON([]byte(files["transcript.json"]))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inputs := &processInputs{Configuration: loaded, Runtime: runtime, Source: source}
|
||||||
|
prepared, err := prepareProcess(context.Background(), processInput{Transcript: filepath.Join(dir, "transcript.json")}, inputs, &bytes.Buffer{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer prepared.Prepared.Output.Close()
|
||||||
|
result, err := executePipeline(context.Background(), inputs, prepared, sourceToCanonicalTranscript(source), func(error) {})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if result.Transcript.Segments[0].Text != "hello" || len(result.Snapshot.Errors) != 1 || result.Snapshot.Errors[0].Scope != "work_unit" {
|
||||||
|
t.Fatalf("result=%+v errors=%+v", result.Transcript, result.Snapshot.Errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
53
internal/cli/process_pipeline.go
Normal file
53
internal/cli/process_pipeline.go
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/processreport"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
func executePipeline(ctx context.Context, loaded *processInputs, prepared *preparedProcess, normalized *schema.Transcript, warn func(error)) (processResult, error) {
|
||||||
|
cfg := &loaded.Configuration.Config
|
||||||
|
specs := contracts.ResolveModuleRunSpecs(cfg)
|
||||||
|
observer := llm.NewObservationCollector()
|
||||||
|
for i := range specs {
|
||||||
|
item := &specs[i]
|
||||||
|
proposal, err := loaded.Runtime.NewRoleClient("proposal", *item.Settings.LLM.ProposalProfile, item.Settings.LLM, observer.Hooks())
|
||||||
|
if err != nil {
|
||||||
|
return processResult{}, fmt.Errorf("%s proposal client: %w", item.InstanceName, err)
|
||||||
|
}
|
||||||
|
validation, err := loaded.Runtime.NewRoleClient("validation", *item.Settings.LLM.ValidationProfile, item.Settings.LLM, observer.Hooks())
|
||||||
|
if err != nil {
|
||||||
|
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
|
||||||
|
}
|
||||||
|
item.ProposalClient, item.ValidationClient = proposal, validation
|
||||||
|
}
|
||||||
|
debugPath := prepared.DebugPath()
|
||||||
|
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{
|
||||||
|
Config: cfg, Transcript: normalized, ModuleSpecs: specs,
|
||||||
|
ProposalDiagnosticsDir: debugPath, ValidationDiagnosticsDir: debugPath,
|
||||||
|
CaptureCandidates: debugPath != "", ReportDiagnosticError: warn,
|
||||||
|
})
|
||||||
|
output.Observations = observer.Snapshot()
|
||||||
|
output.ObservationsAvailable = true
|
||||||
|
output.LLMSummary = llm.Summarize(output.Observations)
|
||||||
|
if debug := prepared.Prepared.Debug; debug != nil {
|
||||||
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactLLMExecution, processreport.BuildLLMExecution(output.Observations)))
|
||||||
|
warn(debug.WriteJSONArtifact(diagnostics.ArtifactCorrectionLedger, processreport.BuildCorrectionLedger(processreport.CorrectionLedgerInput{RunDirectoryPath: debug.Path(), RunOutput: &output})))
|
||||||
|
}
|
||||||
|
var failures []processreport.RunFailure
|
||||||
|
if runErr != nil {
|
||||||
|
failures = append(failures, processreport.RunFailure{Phase: "processing", Reason: outcomes.ReasonInternalFailure})
|
||||||
|
}
|
||||||
|
snapshot := processreport.Build(processreport.BuildInput{RunID: prepared.RunID, Status: reporting.StatusInProgress, StartedAt: prepared.Started, ModuleSpecs: specs, RunOutput: &output, RunFailures: failures})
|
||||||
|
return processResult{Transcript: output.FinalTranscript, Snapshot: snapshot}, runErr
|
||||||
|
}
|
||||||
@@ -24,8 +24,8 @@ type LLMSummary struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type BackendUtilization struct {
|
type BackendUtilization struct {
|
||||||
Concurrency int `json:"concurrency"`
|
Concurrency *int `json:"concurrency,omitempty"`
|
||||||
QueueCapacity int `json:"queue_capacity"`
|
QueueCapacity *int `json:"queue_capacity,omitempty"`
|
||||||
LLMSummary
|
LLMSummary
|
||||||
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
CapacityWaitMS int64 `json:"capacity_wait_ms"`
|
||||||
GenerationMS int64 `json:"generation_ms"`
|
GenerationMS int64 `json:"generation_ms"`
|
||||||
|
|||||||
@@ -1,74 +0,0 @@
|
|||||||
package schema
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
type GlossaryEntry struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Aliases []string `yaml:"aliases,omitempty"`
|
|
||||||
Plural string `yaml:"plural,omitempty"`
|
|
||||||
Category string `yaml:"category"`
|
|
||||||
Summary string `yaml:"summary"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Glossary struct {
|
|
||||||
Entries []GlossaryEntry `yaml:"glossary"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func ParseGlossaryYAML(raw []byte) (*Glossary, error) {
|
|
||||||
var top map[string]any
|
|
||||||
if err := yaml.Unmarshal(raw, &top); err != nil {
|
|
||||||
return nil, &ParseError{Message: fmt.Sprintf("glossary is not valid YAML: %v", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
var g Glossary
|
|
||||||
if err := yaml.Unmarshal(raw, &g); err != nil {
|
|
||||||
return nil, &ParseError{Message: fmt.Sprintf("failed to parse glossary: %v", err)}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := validateGlossary(&g); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &g, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateGlossary(g *Glossary) error {
|
|
||||||
if len(g.Entries) == 0 {
|
|
||||||
return &ValidationError{Field: "glossary", Message: "must contain at least one entry"}
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, e := range g.Entries {
|
|
||||||
entryLabel := fmt.Sprintf("glossary[%d]", i)
|
|
||||||
|
|
||||||
if e.Name == "" {
|
|
||||||
return &ValidationError{Field: fmt.Sprintf("%s.name", entryLabel), Message: "must not be empty"}
|
|
||||||
}
|
|
||||||
|
|
||||||
if e.Category == "" {
|
|
||||||
return &ValidationError{Field: fmt.Sprintf("%s.category", entryLabel), Message: "must not be empty"}
|
|
||||||
}
|
|
||||||
|
|
||||||
if e.Summary == "" {
|
|
||||||
return &ValidationError{Field: fmt.Sprintf("%s.summary", entryLabel), Message: "must not be empty"}
|
|
||||||
}
|
|
||||||
|
|
||||||
for j, alias := range e.Aliases {
|
|
||||||
if alias == "" {
|
|
||||||
return &ValidationError{
|
|
||||||
Field: fmt.Sprintf("%s.aliases[%d]", entryLabel, j),
|
|
||||||
Message: "must not be empty",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if e.Plural != "" && len(e.Plural) == 0 {
|
|
||||||
return &ValidationError{Field: fmt.Sprintf("%s.plural", entryLabel), Message: "must not be empty if present"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
package schema
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_Valid(t *testing.T) {
|
|
||||||
raw, err := os.ReadFile("testdata/glossary_valid.yaml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read test fixture: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
glossary, err := ParseGlossaryYAML(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ParseGlossaryYAML failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(glossary.Entries) != 2 {
|
|
||||||
t.Fatalf("expected 2 entries, got %d", len(glossary.Entries))
|
|
||||||
}
|
|
||||||
|
|
||||||
e1 := glossary.Entries[0]
|
|
||||||
if e1.Name != "Jesters" {
|
|
||||||
t.Errorf("expected entries[0].name = Jesters, got %q", e1.Name)
|
|
||||||
}
|
|
||||||
if e1.Category != "faction" {
|
|
||||||
t.Errorf("expected entries[0].category = faction, got %q", e1.Category)
|
|
||||||
}
|
|
||||||
if e1.Summary != "A faction name." {
|
|
||||||
t.Errorf("expected entries[0].summary = A faction name., got %q", e1.Summary)
|
|
||||||
}
|
|
||||||
if len(e1.Aliases) != 0 {
|
|
||||||
t.Errorf("expected entries[0].aliases = [], got %v", e1.Aliases)
|
|
||||||
}
|
|
||||||
|
|
||||||
e2 := glossary.Entries[1]
|
|
||||||
if e2.Name != "Popov" {
|
|
||||||
t.Errorf("expected entries[1].name = Popov, got %q", e2.Name)
|
|
||||||
}
|
|
||||||
if len(e2.Aliases) != 1 || e2.Aliases[0] != "Hrank" {
|
|
||||||
t.Errorf("expected entries[1].aliases = [Hrank], got %v", e2.Aliases)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_MalformedYAML(t *testing.T) {
|
|
||||||
raw, err := os.ReadFile("testdata/glossary_malformed.yaml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read test fixture: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = ParseGlossaryYAML(raw)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for malformed YAML, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
parseErr, ok := err.(*ParseError)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("expected *ParseError, got %T", err)
|
|
||||||
} else if parseErr.Message == "" {
|
|
||||||
t.Error("expected non-empty parse error message")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_MissingRequiredFields(t *testing.T) {
|
|
||||||
raw, err := os.ReadFile("testdata/glossary_missing_fields.yaml")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read test fixture: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = ParseGlossaryYAML(raw)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for missing required fields, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
valErr, ok := err.(*ValidationError)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("expected *ValidationError, got %T", err)
|
|
||||||
} else if valErr.Field == "" {
|
|
||||||
t.Error("expected non-empty validation error field")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_EmptyGlossary(t *testing.T) {
|
|
||||||
raw := []byte(`glossary: []`)
|
|
||||||
|
|
||||||
_, err := ParseGlossaryYAML(raw)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for empty glossary, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
valErr, ok := err.(*ValidationError)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("expected *ValidationError, got %T", err)
|
|
||||||
} else if valErr.Message != "must contain at least one entry" {
|
|
||||||
t.Errorf("expected 'must contain at least one entry', got %q", valErr.Message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_EmptyName(t *testing.T) {
|
|
||||||
raw := []byte(`
|
|
||||||
glossary:
|
|
||||||
- name: ""
|
|
||||||
category: faction
|
|
||||||
summary: A faction.
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := ParseGlossaryYAML(raw)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for empty name, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
valErr, ok := err.(*ValidationError)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("expected *ValidationError, got %T", err)
|
|
||||||
} else if valErr.Message != "must not be empty" {
|
|
||||||
t.Errorf("expected 'must not be empty', got %q", valErr.Message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseGlossaryYAML_EmptyAlias(t *testing.T) {
|
|
||||||
raw := []byte(`
|
|
||||||
glossary:
|
|
||||||
- name: Test
|
|
||||||
aliases:
|
|
||||||
- ""
|
|
||||||
category: faction
|
|
||||||
summary: A test entry.
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := ParseGlossaryYAML(raw)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected error for empty alias, got nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
valErr, ok := err.(*ValidationError)
|
|
||||||
if !ok {
|
|
||||||
t.Errorf("expected *ValidationError, got %T", err)
|
|
||||||
} else if valErr.Message != "must not be empty" {
|
|
||||||
t.Errorf("expected 'must not be empty', got %q", valErr.Message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
@@ -38,6 +39,13 @@ type ModuleRunSpec struct {
|
|||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
InstanceName string `json:"instance_name"`
|
InstanceName string `json:"instance_name"`
|
||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
|
Index int `json:"-"`
|
||||||
|
ID *string `json:"-"`
|
||||||
|
Settings config.Stage `json:"-"`
|
||||||
|
Glossary []glossary.Entry `json:"-"`
|
||||||
|
ProtectedTerms []glossary.Term `json:"-"`
|
||||||
|
ProposalClient StructuredLLMClient `json:"-"`
|
||||||
|
ValidationClient StructuredLLMClient `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SectionMetadata captures chunk-level metadata without embedding full segment payloads.
|
// SectionMetadata captures chunk-level metadata without embedding full segment payloads.
|
||||||
@@ -64,7 +72,7 @@ func SectionMetadataFromSection(section chunking.Section) SectionMetadata {
|
|||||||
type ExecutionContext struct {
|
type ExecutionContext struct {
|
||||||
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
// ReportDiagnosticError may be called concurrently; it must not affect processing.
|
||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
Config *config.Config `json:"-"`
|
Settings config.Stage `json:"-"`
|
||||||
WorkingTranscript *schema.Transcript `json:"-"`
|
WorkingTranscript *schema.Transcript `json:"-"`
|
||||||
Glossary []glossary.Entry `json:"-"`
|
Glossary []glossary.Entry `json:"-"`
|
||||||
ProtectedTerms []glossary.Term `json:"-"`
|
ProtectedTerms []glossary.Term `json:"-"`
|
||||||
@@ -87,32 +95,20 @@ type ProposalResult struct {
|
|||||||
// ValidationRequest is the input to validator execution.
|
// ValidationRequest is the input to validator execution.
|
||||||
type ValidationRequest = validators.Request
|
type ValidationRequest = validators.Request
|
||||||
|
|
||||||
// ResolveModuleRunSpecs deterministically resolves instance names from logical keys.
|
// ResolveModuleRunSpecs derives isolated consumer views from resolved settings.
|
||||||
// Repeated keys are suffixed with _<n> (1-based), while singleton keys keep their raw key.
|
func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec {
|
||||||
func ResolveModuleRunSpecs(moduleKeys []string) ([]ModuleRunSpec, error) {
|
if cfg == nil {
|
||||||
totals := make(map[string]int, len(moduleKeys))
|
return nil
|
||||||
for i, key := range moduleKeys {
|
|
||||||
if key == "" {
|
|
||||||
return nil, fmt.Errorf("module key at index %d must not be empty", i)
|
|
||||||
}
|
|
||||||
totals[key]++
|
|
||||||
}
|
}
|
||||||
|
specs := make([]ModuleRunSpec, len(cfg.Pipeline.Stages))
|
||||||
seen := make(map[string]int, len(totals))
|
for i, item := range cfg.Pipeline.Stages {
|
||||||
specs := make([]ModuleRunSpec, len(moduleKeys))
|
selected := glossary.Select(cfg.GlossaryCollections, item.Glossaries)
|
||||||
for i, key := range moduleKeys {
|
protected := glossary.Select(cfg.GlossaryCollections, item.ProtectedTerms)
|
||||||
seen[key]++
|
var excluded []glossary.Entry
|
||||||
|
if item.Module == modulecatalog.KeyGlossary {
|
||||||
name := key
|
excluded = selected
|
||||||
if totals[key] > 1 {
|
|
||||||
name = fmt.Sprintf("%s_%d", key, seen[key])
|
|
||||||
}
|
|
||||||
|
|
||||||
specs[i] = ModuleRunSpec{
|
|
||||||
ModuleKey: key,
|
|
||||||
InstanceName: name,
|
|
||||||
}
|
}
|
||||||
|
specs[i] = ModuleRunSpec{ModuleKey: item.Module, InstanceName: fmt.Sprintf("%s_%d", item.Module, i+1), Index: i, ID: item.ID, Settings: item, Glossary: selected, ProtectedTerms: glossary.Vocabulary(protected, excluded)}
|
||||||
}
|
}
|
||||||
|
return specs
|
||||||
return specs, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,9 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
glossarymodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
|
glossarymodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
|
||||||
grammarmodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
grammarmodule "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||||
@@ -32,19 +30,9 @@ func IsKnownModuleKey(key string) bool {
|
|||||||
return modulecatalog.IsSupported(key)
|
return modulecatalog.IsSupported(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dependencies holds explicit constructor dependencies for module creation.
|
// ConstructRequest identifies the module to construct.
|
||||||
type Dependencies struct {
|
|
||||||
Config *config.Config
|
|
||||||
Glossary *schema.Glossary
|
|
||||||
ProposalLLMClient contracts.StructuredLLMClient
|
|
||||||
ValidationLLMClient contracts.StructuredLLMClient
|
|
||||||
DiagnosticsDir string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConstructRequest is one module-construction request.
|
|
||||||
type ConstructRequest struct {
|
type ConstructRequest struct {
|
||||||
RunSpec contracts.ModuleRunSpec
|
RunSpec contracts.ModuleRunSpec
|
||||||
Dependencies
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Constructor builds one module instance from a run spec and explicit deps.
|
// Constructor builds one module instance from a run spec and explicit deps.
|
||||||
@@ -52,14 +40,12 @@ type Constructor func(ctx context.Context, req ConstructRequest) (contracts.Tran
|
|||||||
|
|
||||||
// Factory resolves configured module specs into module instances.
|
// Factory resolves configured module specs into module instances.
|
||||||
type Factory struct {
|
type Factory struct {
|
||||||
deps Dependencies
|
|
||||||
constructors map[string]Constructor
|
constructors map[string]Constructor
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFactory creates a production module registry.
|
// NewFactory creates a production module registry.
|
||||||
func NewFactory(deps Dependencies) *Factory {
|
func NewFactory() *Factory {
|
||||||
factory := &Factory{
|
factory := &Factory{
|
||||||
deps: deps,
|
|
||||||
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
|
constructors: make(map[string]Constructor, len(modulecatalog.SupportedKeys())),
|
||||||
}
|
}
|
||||||
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
_ = factory.RegisterConstructor(ModuleKeyGlossary, constructGlossaryModule)
|
||||||
@@ -125,16 +111,7 @@ func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Transcr
|
|||||||
return nil, fmt.Errorf("internal module registry error: constructor for module %q is not configured", key)
|
return nil, fmt.Errorf("internal module registry error: constructor for module %q is not configured", key)
|
||||||
}
|
}
|
||||||
|
|
||||||
module, err := constructor(context.Background(), ConstructRequest{
|
module, err := constructor(context.Background(), ConstructRequest{RunSpec: spec})
|
||||||
RunSpec: spec,
|
|
||||||
Dependencies: Dependencies{
|
|
||||||
Config: f.deps.Config,
|
|
||||||
Glossary: f.deps.Glossary,
|
|
||||||
ProposalLLMClient: f.deps.ProposalLLMClient,
|
|
||||||
ValidationLLMClient: f.deps.ValidationLLMClient,
|
|
||||||
DiagnosticsDir: f.deps.DiagnosticsDir,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("construct module %q: %w", spec.InstanceName, err)
|
return nil, fmt.Errorf("construct module %q: %w", spec.InstanceName, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,11 +7,14 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Input contains already acquired application and transcript inputs.
|
// Input contains already acquired application and transcript inputs.
|
||||||
type Input struct {
|
type Input struct {
|
||||||
|
InspectProfile func(context.Context, string) error
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Transcript *schema.SourceTranscript
|
Transcript *schema.SourceTranscript
|
||||||
Output diagnostics.OutputSelection
|
Output diagnostics.OutputSelection
|
||||||
@@ -45,6 +48,9 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
|
|||||||
if input.Transcript == nil {
|
if input.Transcript == nil {
|
||||||
return nil, fmt.Errorf("transcript is required")
|
return nil, fmt.Errorf("transcript is required")
|
||||||
}
|
}
|
||||||
|
if err := CheckReadiness(ctx, input.Config, input.InspectProfile); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
session, err := diagnostics.SelectOutput(input.Output)
|
session, err := diagnostics.SelectOutput(input.Output)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -74,3 +80,38 @@ func Prepare(ctx context.Context, input Input) (*Prepared, error) {
|
|||||||
}
|
}
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CheckReadiness checks all resolved processing selections before output preparation.
|
||||||
|
func CheckReadiness(ctx context.Context, cfg *config.Config, inspectProfile func(context.Context, string) error) error {
|
||||||
|
if cfg == nil {
|
||||||
|
return fmt.Errorf("application configuration is required")
|
||||||
|
}
|
||||||
|
label := func(index int, item config.Stage) string {
|
||||||
|
if item.ID != nil {
|
||||||
|
return fmt.Sprintf("stage %d (%s, id %q)", index+1, item.Module, *item.ID)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("stage %d (%s)", index+1, item.Module)
|
||||||
|
}
|
||||||
|
for index, item := range cfg.Pipeline.Stages {
|
||||||
|
if item.Module == modulecatalog.KeyGlossary && len(glossary.Select(cfg.GlossaryCollections, item.Glossaries)) == 0 {
|
||||||
|
return fmt.Errorf("%s has no glossary entries in selected collections %q", label(index, item), item.Glossaries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for index, item := range cfg.Pipeline.Stages {
|
||||||
|
for _, selection := range []struct {
|
||||||
|
role string
|
||||||
|
profile *string
|
||||||
|
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}} {
|
||||||
|
if selection.profile == nil || *selection.profile == "" {
|
||||||
|
return fmt.Errorf("%s requires a %s profile", label(index, item), selection.role)
|
||||||
|
}
|
||||||
|
if inspectProfile == nil {
|
||||||
|
return fmt.Errorf("profile inspection is required")
|
||||||
|
}
|
||||||
|
if err := inspectProfile(ctx, *selection.profile); err != nil {
|
||||||
|
return fmt.Errorf("%s %s profile %q: %w", label(index, item), selection.role, *selection.profile, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|||||||
39
internal/framework/preflight/readiness_test.go
Normal file
39
internal/framework/preflight/readiness_test.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
package preflight
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/diagnostics"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLaterEmptyGlossaryFailsBeforeOutputPreparation(t *testing.T) {
|
||||||
|
profile, id := "selected", "spells"
|
||||||
|
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{
|
||||||
|
{Module: "grammar", LLM: config.LLM{ProposalProfile: &profile, ValidationProfile: &profile}},
|
||||||
|
{Module: "glossary", ID: &id, Glossaries: []string{"empty"}, LLM: config.LLM{ProposalProfile: &profile, ValidationProfile: &profile}},
|
||||||
|
}}}
|
||||||
|
output := filepath.Join(t.TempDir(), "transcript.json")
|
||||||
|
calls := 0
|
||||||
|
_, err := Prepare(context.Background(), Input{Config: cfg, Transcript: &schema.SourceTranscript{}, Output: diagnostics.OutputSelection{TranscriptPath: output}, InspectProfile: func(context.Context, string) error { calls++; return nil }})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "stage 2") || !strings.Contains(err.Error(), "spells") || !strings.Contains(err.Error(), "empty") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if calls != 0 {
|
||||||
|
t.Fatal("inspected profiles before checking all glossary selections")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(output); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("output exists: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyPipelineNeedsNoProfile(t *testing.T) {
|
||||||
|
if err := CheckReadiness(context.Background(), &config.Config{}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,12 +3,11 @@ package processreport
|
|||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
pk "go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||||
@@ -23,7 +22,7 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// BuildUtilization groups complete logical calls by their selected backend.
|
// BuildUtilization groups complete logical calls by their selected backend.
|
||||||
func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backend) reporting.UtilizationDiagnostics {
|
func BuildUtilization(output *runner.RunOutput, backends map[string]*pk.BackendInspection) reporting.UtilizationDiagnostics {
|
||||||
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
||||||
if output == nil {
|
if output == nil {
|
||||||
return result
|
return result
|
||||||
@@ -85,7 +84,7 @@ func BuildUtilization(output *runner.RunOutput, backends map[string]config.Backe
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
result.Backends[id] = reporting.BackendUtilization{
|
result.Backends[id] = reporting.BackendUtilization{
|
||||||
Concurrency: backend.Concurrency, QueueCapacity: backend.QueueCapacity,
|
Concurrency: backend.ConcurrencyLimit, QueueCapacity: backend.QueueCapacity,
|
||||||
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
|
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
|
||||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
@@ -57,7 +56,6 @@ type Request struct {
|
|||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
WorkingTranscript *schema.Transcript `json:"-"`
|
WorkingTranscript *schema.Transcript `json:"-"`
|
||||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||||
Config *config.Config `json:"-"`
|
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
Variables map[string]string `json:"variables"`
|
Variables map[string]string `json:"variables"`
|
||||||
StageName string `json:"stage_name,omitempty"`
|
StageName string `json:"stage_name,omitempty"`
|
||||||
|
|||||||
@@ -35,10 +35,7 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
|||||||
sectionIndex = req.ProposalRequest.Section.Index
|
sectionIndex = req.ProposalRequest.Section.Index
|
||||||
}
|
}
|
||||||
|
|
||||||
transcriptDescription := ""
|
transcriptDescription := req.ProposalRequest.Settings.Context.Description
|
||||||
if req.ProposalRequest.Config != nil {
|
|
||||||
transcriptDescription = req.ProposalRequest.Config.Context.Description
|
|
||||||
}
|
|
||||||
|
|
||||||
variables, err := req.BuildVariables(
|
variables, err := req.BuildVariables(
|
||||||
req.ProposalRequest.WorkingTranscript,
|
req.ProposalRequest.WorkingTranscript,
|
||||||
@@ -61,7 +58,6 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
|||||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||||
Section: req.ProposalRequest.Section,
|
Section: req.ProposalRequest.Section,
|
||||||
Config: req.ProposalRequest.Config,
|
|
||||||
PromptID: promptMetadata.PromptID,
|
PromptID: promptMetadata.PromptID,
|
||||||
Variables: variables,
|
Variables: variables,
|
||||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||||
|
|||||||
67
internal/framework/runner/ordered_workflow_test.go
Normal file
67
internal/framework/runner/ordered_workflow_test.go
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
)
|
||||||
|
|
||||||
|
type orderedFactory struct {
|
||||||
|
seen chan contracts.ProposalRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f orderedFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
|
||||||
|
return orderedModule{spec: spec, seen: f.seen}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type orderedModule struct {
|
||||||
|
spec contracts.ModuleRunSpec
|
||||||
|
seen chan contracts.ProposalRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m orderedModule) Key() string { return m.spec.ModuleKey }
|
||||||
|
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||||
|
return proposals.ReplacementPolicyRequireUnique
|
||||||
|
}
|
||||||
|
func (m orderedModule) Validators() []contracts.Validator { return nil }
|
||||||
|
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||||
|
m.seen <- req
|
||||||
|
original, corrected := "party", "Party"
|
||||||
|
if m.spec.Index == 1 {
|
||||||
|
original, corrected = "spell", "Spell"
|
||||||
|
}
|
||||||
|
if m.spec.Index == 2 {
|
||||||
|
original, corrected = "Party Spell", "Party Spell!"
|
||||||
|
}
|
||||||
|
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: original, CorrectedText: corrected, Confidence: 1}}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOrderedWorkflowUsesIndependentSelectionsAndPriorEdits(t *testing.T) {
|
||||||
|
cfg := &config.Config{GlossaryCollections: map[string]config.GlossaryCollection{
|
||||||
|
"party": {Entries: []config.GlossaryEntry{{ID: "party", Name: "Party", Category: "person"}}},
|
||||||
|
"spells": {Entries: []config.GlossaryEntry{{ID: "spell", Name: "Spell", Category: "spell"}}},
|
||||||
|
}, Pipeline: config.Pipeline{Stages: []config.Stage{
|
||||||
|
{Module: "glossary", Glossaries: []string{"party"}, ProtectedTerms: []string{"party", "spells"}, Context: config.Context{Description: "party context"}},
|
||||||
|
{Module: "glossary", Glossaries: []string{"spells"}, ProtectedTerms: []string{"party", "spells"}, Context: config.Context{Description: "spell context"}},
|
||||||
|
{Module: "grammar", Glossaries: []string{"party", "spells"}, ProtectedTerms: []string{"party", "spells"}},
|
||||||
|
}}}
|
||||||
|
seen := make(chan contracts.ProposalRequest, 3)
|
||||||
|
output, err := New(orderedFactory{seen: seen}).Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "party spell"}}}, ModuleSpecs: contracts.ResolveModuleRunSpecs(cfg)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := output.FinalTranscript.Segments[0].Text; got != "Party Spell!" {
|
||||||
|
t.Fatalf("final text = %q", got)
|
||||||
|
}
|
||||||
|
first, second, third := <-seen, <-seen, <-seen
|
||||||
|
if first.Settings.Context.Description != "party context" || second.Settings.Context.Description != "spell context" || len(first.Glossary) != 1 || first.Glossary[0].Name != "Party" || len(second.Glossary) != 1 || second.Glossary[0].Name != "Spell" || len(third.Glossary) != 2 {
|
||||||
|
t.Fatalf("incorrect context: %+v %+v %+v", first, second, third)
|
||||||
|
}
|
||||||
|
if second.WorkingTranscript.Segments[0].Text != "Party spell" || third.WorkingTranscript.Segments[0].Text != "Party Spell" {
|
||||||
|
t.Fatalf("edits not carried forward: %q, %q", second.WorkingTranscript.Segments[0].Text, third.WorkingTranscript.Segments[0].Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||||
@@ -93,11 +94,8 @@ type RunInput struct {
|
|||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Transcript *schema.Transcript
|
Transcript *schema.Transcript
|
||||||
Glossary *schema.Glossary
|
|
||||||
ModuleSpecs []contracts.ModuleRunSpec
|
ModuleSpecs []contracts.ModuleRunSpec
|
||||||
ProposalLLMClient contracts.StructuredLLMClient
|
|
||||||
ProposalDiagnosticsDir string
|
ProposalDiagnosticsDir string
|
||||||
ValidationLLMClient contracts.StructuredLLMClient
|
|
||||||
ValidationDiagnosticsDir string
|
ValidationDiagnosticsDir string
|
||||||
CaptureCandidates bool
|
CaptureCandidates bool
|
||||||
timings *validatorTimingCollector
|
timings *validatorTimingCollector
|
||||||
@@ -185,12 +183,10 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
Module: module,
|
Module: module,
|
||||||
Spec: spec,
|
Spec: spec,
|
||||||
Policy: policy,
|
Policy: policy,
|
||||||
Config: input.Config,
|
|
||||||
Glossary: input.Glossary,
|
|
||||||
Sections: sections,
|
Sections: sections,
|
||||||
ProposalClient: input.ProposalLLMClient,
|
ProposalClient: spec.ProposalClient,
|
||||||
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
DiagnosticsDir: input.ProposalDiagnosticsDir,
|
||||||
ValidationClient: input.ValidationLLMClient,
|
ValidationClient: spec.ValidationClient,
|
||||||
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
|
||||||
Timings: input.timings,
|
Timings: input.timings,
|
||||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||||
@@ -220,7 +216,7 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
|
|||||||
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
|
return snapshot(), fmt.Errorf("module %q failed: %w", spec.InstanceName, pipelineErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy)
|
updated, applied, skipped, appliedBySection, applyErr := applyWithinSections(sections, pipelineResult.Approved, policy, validators.NewProtectedVocabulary(spec.ProtectedTerms), spec.ModuleKey == modulecatalog.KeySpokenWord)
|
||||||
if applyErr != nil {
|
if applyErr != nil {
|
||||||
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
|
return snapshot(), fmt.Errorf("module %q application failed: %w", spec.InstanceName, applyErr)
|
||||||
}
|
}
|
||||||
@@ -268,7 +264,7 @@ func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
|
|||||||
|
|
||||||
// applyWithinSections leaves section membership and initial metadata unchanged.
|
// applyWithinSections leaves section membership and initial metadata unchanged.
|
||||||
// Approved proposals are applied only to their identified section.
|
// Approved proposals are applied only to their identified section.
|
||||||
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary validators.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
||||||
bySection := make(map[int][]proposals.EnrichedCorrectionProposal, len(sections))
|
bySection := make(map[int][]proposals.EnrichedCorrectionProposal, len(sections))
|
||||||
known := make(map[int]struct{}, len(sections))
|
known := make(map[int]struct{}, len(sections))
|
||||||
for _, section := range sections {
|
for _, section := range sections {
|
||||||
@@ -289,7 +285,7 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
|
|||||||
skipped := make([]proposals.SkippedChange, 0)
|
skipped := make([]proposals.SkippedChange, 0)
|
||||||
appliedBySection := make([]int, len(sections))
|
appliedBySection := make([]int, len(sections))
|
||||||
for i, section := range sections {
|
for i, section := range sections {
|
||||||
result := proposals.ApplyProposals(transcriptFromSection(section), bySection[section.Index], policy)
|
result := proposals.ApplyProposals(transcriptFromSection(section), bySection[section.Index], policy, func(before, after string) bool { return vocabulary.AllowsChange(before, after, allowRepetitionCleanup) })
|
||||||
updated[i].Segments = result.Transcript.Segments
|
updated[i].Segments = result.Transcript.Segments
|
||||||
applied = append(applied, result.Applied...)
|
applied = append(applied, result.Applied...)
|
||||||
skipped = append(skipped, result.Skipped...)
|
skipped = append(skipped, result.Skipped...)
|
||||||
@@ -304,8 +300,6 @@ type collectSectionProposalsInput struct {
|
|||||||
Module contracts.TranscriptModule
|
Module contracts.TranscriptModule
|
||||||
Spec contracts.ModuleRunSpec
|
Spec contracts.ModuleRunSpec
|
||||||
Policy proposals.ReplacementPolicy
|
Policy proposals.ReplacementPolicy
|
||||||
Config *config.Config
|
|
||||||
Glossary *schema.Glossary
|
|
||||||
Sections []chunking.Section
|
Sections []chunking.Section
|
||||||
ProposalClient contracts.StructuredLLMClient
|
ProposalClient contracts.StructuredLLMClient
|
||||||
DiagnosticsDir string
|
DiagnosticsDir string
|
||||||
@@ -364,9 +358,10 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
|||||||
meta := contracts.SectionMetadataFromSection(section)
|
meta := contracts.SectionMetadataFromSection(section)
|
||||||
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
proposalResult, err := input.Module.Propose(runCtx, contracts.ProposalRequest{
|
||||||
ExecutionContext: contracts.ExecutionContext{
|
ExecutionContext: contracts.ExecutionContext{
|
||||||
Config: input.Config,
|
Settings: input.Spec.Settings,
|
||||||
WorkingTranscript: transcriptFromSection(section),
|
WorkingTranscript: transcriptFromSection(section),
|
||||||
Glossary: input.Glossary,
|
Glossary: input.Spec.Glossary,
|
||||||
|
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||||
Section: &meta,
|
Section: &meta,
|
||||||
DiagnosticsDir: input.DiagnosticsDir,
|
DiagnosticsDir: input.DiagnosticsDir,
|
||||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||||
@@ -506,8 +501,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
|||||||
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
|
validated, err := validateSectionCandidates(runCtx, validateSectionCandidatesInput{
|
||||||
Spec: input.Spec,
|
Spec: input.Spec,
|
||||||
Policy: input.Policy,
|
Policy: input.Policy,
|
||||||
Glossary: input.Glossary,
|
|
||||||
Config: input.Config,
|
|
||||||
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
WorkingTranscript: transcriptFromSection(sectionResult.section),
|
||||||
SectionIndex: §ionMetadata.Index,
|
SectionIndex: §ionMetadata.Index,
|
||||||
Validators: validatorsOrdered,
|
Validators: validatorsOrdered,
|
||||||
@@ -592,8 +585,6 @@ type validateSectionCandidatesInput struct {
|
|||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
Spec contracts.ModuleRunSpec
|
Spec contracts.ModuleRunSpec
|
||||||
Policy proposals.ReplacementPolicy
|
Policy proposals.ReplacementPolicy
|
||||||
Glossary *schema.Glossary
|
|
||||||
Config *config.Config
|
|
||||||
WorkingTranscript *schema.Transcript
|
WorkingTranscript *schema.Transcript
|
||||||
SectionIndex *int
|
SectionIndex *int
|
||||||
Validators []contracts.Validator
|
Validators []contracts.Validator
|
||||||
@@ -624,10 +615,16 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
}
|
}
|
||||||
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
var diagnosticsWriter validators.InteractionDiagnosticsWriter
|
||||||
if input.DiagnosticsDir != "" {
|
if input.DiagnosticsDir != "" {
|
||||||
|
var secrets []string
|
||||||
|
if source, ok := input.ValidationLLMClient.(interface {
|
||||||
|
ConfiguredSecrets(context.Context) []string
|
||||||
|
}); ok {
|
||||||
|
secrets = source.ConfiguredSecrets(ctx)
|
||||||
|
}
|
||||||
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
diagnosticsWriter = &llmDiagnosticsWriterAdapter{
|
||||||
writer: llm.NewDiagnosticsWriter(
|
writer: llm.NewDiagnosticsWriter(
|
||||||
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
filepath.Join(input.DiagnosticsDir, input.Spec.InstanceName),
|
||||||
llm.ConfiguredSecrets(input.Config),
|
secrets,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -641,8 +638,9 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
|||||||
SectionIndex: input.SectionIndex,
|
SectionIndex: input.SectionIndex,
|
||||||
ValidatorOrder: &validatorOrder,
|
ValidatorOrder: &validatorOrder,
|
||||||
ReplacementPolicy: input.Policy,
|
ReplacementPolicy: input.Policy,
|
||||||
Glossary: input.Glossary,
|
Glossary: input.Spec.Glossary,
|
||||||
Config: input.Config,
|
ProtectedTerms: input.Spec.ProtectedTerms,
|
||||||
|
Settings: input.Spec.Settings,
|
||||||
LLMClient: input.ValidationLLMClient,
|
LLMClient: input.ValidationLLMClient,
|
||||||
DiagnosticsWriter: diagnosticsWriter,
|
DiagnosticsWriter: diagnosticsWriter,
|
||||||
ReportDiagnosticError: input.ReportDiagnosticError,
|
ReportDiagnosticError: input.ReportDiagnosticError,
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ type ConfidenceThresholdValidator struct{}
|
|||||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||||
|
|
||||||
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
|
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||||
threshold := confidenceThresholdForModule(req.ModuleKey, req.Config)
|
threshold := req.Settings.ConfidenceThreshold
|
||||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||||
for _, c := range req.CandidateProposal {
|
for _, c := range req.CandidateProposal {
|
||||||
if c.Confidence < threshold {
|
if c.Confidence < threshold {
|
||||||
|
|||||||
@@ -72,8 +72,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
}
|
}
|
||||||
|
|
||||||
maxTokens := config.DefaultValidationMaxPromptTokens
|
maxTokens := config.DefaultValidationMaxPromptTokens
|
||||||
if req.Config != nil && req.Config.Validation.MaxPromptTokens > 0 {
|
if req.Settings.Validation.MaxPromptTokens > 0 {
|
||||||
maxTokens = req.Config.Validation.MaxPromptTokens
|
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := range validationReq.Items {
|
for i := range validationReq.Items {
|
||||||
@@ -131,9 +131,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
|||||||
return partialResult(), err
|
return partialResult(), err
|
||||||
}
|
}
|
||||||
transcriptDescription := ""
|
transcriptDescription := ""
|
||||||
if req.Config != nil {
|
transcriptDescription = req.Settings.Context.Description
|
||||||
transcriptDescription = req.Config.Context.Description
|
|
||||||
}
|
|
||||||
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
|
variables, err := BuildValidationVariables(batch.Items, transcriptDescription)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return partialResult(), err
|
return partialResult(), err
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
@@ -44,7 +43,7 @@ type Request struct {
|
|||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
Glossary []glossary.Entry `json:"-"`
|
Glossary []glossary.Entry `json:"-"`
|
||||||
ProtectedTerms []glossary.Term `json:"-"`
|
ProtectedTerms []glossary.Term `json:"-"`
|
||||||
Config *config.Config `json:"-"`
|
Settings config.Stage `json:"-"`
|
||||||
LLMClient StructuredLLMClient `json:"-"`
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
||||||
}
|
}
|
||||||
@@ -142,21 +141,3 @@ func rejection(index int, reasonCode string, msg string) Decision {
|
|||||||
func unavailable(index int, reasonCode string, msg string) Decision {
|
func unavailable(index int, reasonCode string, msg string) Decision {
|
||||||
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionUnavailable, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
|
return Decision{ProposalIndex: index, Approved: false, Disposition: DispositionUnavailable, ReasonCode: reasonCode, Message: strings.TrimSpace(msg)}
|
||||||
}
|
}
|
||||||
|
|
||||||
func confidenceThresholdForModule(moduleKey string, cfg *config.Config) float64 {
|
|
||||||
if cfg == nil {
|
|
||||||
return 0.0
|
|
||||||
}
|
|
||||||
switch moduleKey {
|
|
||||||
case modulecatalog.KeyGlossary:
|
|
||||||
return cfg.Thresholds.Glossary
|
|
||||||
case modulecatalog.KeyGrammar:
|
|
||||||
return cfg.Thresholds.Grammar
|
|
||||||
case modulecatalog.KeyHomophones:
|
|
||||||
return cfg.Thresholds.Homophones
|
|
||||||
case modulecatalog.KeySpokenWord:
|
|
||||||
return cfg.Thresholds.SpokenWord
|
|
||||||
default:
|
|
||||||
return 0.0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user