Remove retired reporting and diagnostic code

This commit is contained in:
2026-11-10 20:26:43 +00:00
parent a0e45c8f92
commit 9a52c8c246
21 changed files with 59 additions and 706 deletions

View File

@@ -4,9 +4,6 @@ import (
"context"
"errors"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -22,38 +19,6 @@ func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
return nil, errors.New("provider access forbidden")
}
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
data := `version: 3
glossary_collections:
common:
entries:
- id: a
name: secret-value-not-for-explanation
category: character
derived:
extends: common
`
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(data), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(file)}, "")
if err != nil {
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
}
explanation, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
text := string(explanation)
if strings.Contains(text, "secret-value-not-for-explanation") {
t.Fatal("explanation exposed an effective value")
}
if !strings.Contains(text, "/glossary_collections/derived/entries") || !strings.Contains(text, "inheritance:") || !strings.Contains(text, "/glossary_collections/common/entries") {
t.Fatalf("inherited field evidence missing: %s", text)
}
}
func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
@@ -62,7 +27,4 @@ func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
if _, err := ExplanationYAML(nil); err == nil {
t.Fatal("nil configuration accepted")
}
}

View File

@@ -1,80 +0,0 @@
package llm
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
"gopkg.in/yaml.v3"
)
type derivationView struct {
Operation ck.ChangeKind `yaml:"operation"`
Reason string `yaml:"reason"`
Dependencies []string `yaml:"dependencies,omitempty"`
}
type inheritanceStepView struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
}
type explanationView struct {
Path string `yaml:"path"`
Source string `yaml:"source"`
Layer ck.Layer `yaml:"layer"`
Derivations []derivationView `yaml:"derivations,omitempty"`
Inheritance []inheritanceStepView `yaml:"inheritance,omitempty"`
}
// ExplanationYAML reports only origins and recorded resolution operations.
// It intentionally excludes effective and historical values, including secrets.
func ExplanationYAML(result *ck.Result[config.Config]) ([]byte, error) {
if result == nil {
return nil, fmt.Errorf("configuration result is unavailable")
}
metadata := result.Metadata()
sources := make(map[string]string)
for _, source := range result.Sources() {
sources[source.ID] = source.Label
}
records := make([]explanationView, 0)
paths := metadata.HistoricalPaths()
for i, path := range paths {
if path.String() == "" {
continue
}
if i+1 < len(paths) && strings.HasPrefix(paths[i+1].String(), path.String()+"/") {
continue
}
info, ok := metadata.Lookup(path)
if !ok {
continue
}
record := explanationView{Path: path.String(), Layer: info.Origin.Layer, Source: string(info.Origin.Layer)}
for _, step := range info.Origin.Inheritance {
record.Inheritance = append(record.Inheritance, inheritanceStepView{
From: step.From.String(), To: step.To.String(), Reference: step.Reference.String(),
})
}
if len(info.Origin.Locations) > 0 {
if label := sources[info.Origin.Locations[0].SourceID]; label != "" {
record.Source = label
}
}
for _, change := range metadata.History(path) {
if change.Origin.Layer != ck.LayerResolution {
continue
}
for _, reason := range change.Reasons {
derivation := derivationView{Operation: change.Origin.Operation, Reason: reason}
for _, dependency := range change.Dependencies {
derivation.Dependencies = append(derivation.Dependencies, dependency.String())
}
record.Derivations = append(record.Derivations, derivation)
}
}
records = append(records, record)
}
return yaml.Marshal(records)
}

View File

@@ -5,9 +5,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
@@ -21,81 +19,6 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
}
}
// BuildUtilization groups complete logical calls by their selected backend.
func BuildUtilization(output *runner.RunOutput, backends map[string]*pk.BackendInspection) reporting.UtilizationDiagnostics {
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
if output == nil {
return result
}
result.RunWallTimeMS = output.RunWallTimeMS
groups := make(map[string]*llm.ObservationSnapshot, len(backends))
callBackends := make(map[uint64]string, len(output.Observations.Calls))
moduleIndices := make(map[string]int)
moduleIndex := func(key, instance string) int {
if i, ok := moduleIndices[instance]; ok {
return i
}
i := len(result.Modules)
moduleIndices[instance] = i
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: key, ModuleInstance: instance})
return i
}
// Seed configured order, including modules that made no LLM calls.
for _, module := range output.ModuleResults {
i := moduleIndex(module.ModuleKey, module.ModuleInstance)
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
}
for _, call := range output.Observations.Calls {
id := call.Prompt.BackendID
if groups[id] == nil {
groups[id] = &llm.ObservationSnapshot{}
}
groups[id].Calls = append(groups[id].Calls, call)
callBackends[call.CallID] = id
if call.Context.ModuleInstance != "" {
i := moduleIndex(call.Context.ModuleKey, call.Context.ModuleInstance)
switch call.Context.Role {
case "proposal":
result.Modules[i].ProposalLLMCalls++
case "validation":
result.Modules[i].ValidationLLMCalls++
}
}
}
for _, event := range output.Observations.Events {
if id, ok := callBackends[event.CallID]; ok {
groups[id].Events = append(groups[id].Events, event)
}
}
for id, backend := range backends {
group := groups[id]
if group == nil {
group = &llm.ObservationSnapshot{}
}
var wait, generation, retry, wall time.Duration
for _, call := range group.Calls {
wall += call.WallDuration
}
for _, event := range group.Events {
if event.Kind == string(pk.ExecutionFinished) {
wait += event.CapacityWait
generation += event.GenerationDuration
retry += event.RetryDelay
}
}
result.Backends[id] = reporting.BackendUtilization{
Concurrency: backend.ConcurrencyLimit, QueueCapacity: backend.QueueCapacity,
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
}
}
for _, timing := range output.ValidatorTimings {
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
}
return result
}
type executionCall struct {
CallID uint64 `json:"call_id"`
Context structuredoutput.DiagnosticContext `json:"context"`

View File

@@ -8,12 +8,11 @@ import (
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "go.maximumdirect.net/promptkit"
)
func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
start := time.Now().UTC()
snapshot := llm.ObservationSnapshot{
Calls: []llm.LogicalCall{
@@ -22,14 +21,6 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
},
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
}
output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"primary": {ConcurrencyLimit: intPointer(2), QueueCapacity: intPointer(3)}, "secondary": {ConcurrencyLimit: intPointer(1)}})
if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
}
if utilization.Backends["secondary"].LogicalCalls != 1 || utilization.Backends["secondary"].ProviderAttempts != 0 || !utilization.Backends["secondary"].Usage.Total.Known {
t.Fatalf("predispatch backend: %+v", utilization.Backends["secondary"])
}
raw, err := json.Marshal(BuildLLMExecution(snapshot))
if err != nil {
t.Fatal(err)
@@ -57,17 +48,11 @@ func BenchmarkExecutionReports(b *testing.B) {
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
}
output := &runner.RunOutput{Observations: snapshot}
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
BuildLLMExecution(snapshot)
}
})
b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
BuildUtilization(output, map[string]*pk.BackendInspection{"local": {ConcurrencyLimit: intPointer(1)}})
}
})
}
}
@@ -108,15 +93,4 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
if execution.Calls[1].Summary.Usage.Total.Value != 7 || !execution.Calls[1].Summary.Usage.Total.Complete || execution.Calls[0].Summary.Usage.Total.Known {
t.Fatalf("call usage: %+v", execution.Calls)
}
output := &runner.RunOutput{Observations: snapshot}
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"a": {}, "b": {}})
total := utilization.Backends["a"].Usage.Total
if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
t.Fatalf("partial backend usage: %+v", utilization)
}
if len(utilization.Modules) != 2 || utilization.Modules[1].ProposalLLMCalls != 1 || utilization.Modules[1].ValidationLLMCalls != 1 {
t.Fatalf("module grouping: %+v", utilization.Modules)
}
}
func intPointer(value int) *int { return &value }

View File

@@ -25,7 +25,6 @@ type BuildInput struct {
StartedAt time.Time
CompletedAt *time.Time
TranscriptPath string
GlossaryPath string
OutputPath string
ModuleSpecs []contracts.ModuleRunSpec
RunOutput *runner.RunOutput
@@ -36,7 +35,7 @@ type BuildInput struct {
func Build(input BuildInput) reporting.Snapshot {
snapshot := reporting.Snapshot{
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
TranscriptPath: input.TranscriptPath, GlossaryPath: input.GlossaryPath, OutputPath: input.OutputPath,
TranscriptPath: input.TranscriptPath, OutputPath: input.OutputPath,
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
}

View File

@@ -7,7 +7,6 @@ import (
"path/filepath"
"sort"
"sync"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -51,8 +50,6 @@ type ModuleResult struct {
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
SkippedChanges []proposals.SkippedChange `json:"skipped_changes,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
}
// WorkUnitOutcome keeps initial section identity while text evolves between
@@ -102,22 +99,16 @@ type RunInput struct {
ProposalDiagnosticsDir string
ValidationDiagnosticsDir string
CaptureCandidates bool
timings *validatorTimingCollector
}
// RunOutput is the deterministic runner output.
type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
InventoryAvailable bool `json:"-"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
Observations llm.ObservationSnapshot `json:"-"`
ObservationsAvailable bool `json:"-"`
LLMSummary llm.ExecutionSummary `json:"-"`
RunWallTimeMS int64 `json:"-"`
ValidatorTimings []ValidatorTiming `json:"-"`
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
InventoryAvailable bool `json:"-"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
}
func New(factory ModuleFactory) *Runner {
@@ -125,15 +116,6 @@ func New(factory ModuleFactory) *Runner {
}
func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
started := time.Now()
input.timings = newValidatorTimingCollector()
output, err := r.run(ctx, input)
output.RunWallTimeMS = time.Since(started).Milliseconds()
output.ValidatorTimings = input.timings.snapshot()
return output, err
}
func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
if r == nil || r.factory == nil {
return RunOutput{}, fmt.Errorf("runner module factory is required")
}
@@ -167,7 +149,6 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
}
for moduleIndex, spec := range input.ModuleSpecs {
startedAt := time.Now().UTC()
module, err := r.factory.ModuleForSpec(spec)
if err != nil {
failed := ModuleResult{
@@ -176,8 +157,6 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ModuleInstance: spec.InstanceName,
Status: ModuleStatusFailed,
ErrorMessage: err.Error(),
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
return snapshot(), fmt.Errorf("module %q setup failed: %w", spec.InstanceName, err)
@@ -193,7 +172,6 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
DiagnosticsDir: input.ProposalDiagnosticsDir,
ValidationClient: spec.ValidationClient,
ValidationDiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.timings,
ReportDiagnosticError: input.ReportDiagnosticError,
})
if input.CaptureCandidates {
@@ -211,8 +189,6 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidatorDecisions: pipelineResult.ValidatorDecisions,
ValidatorRejected: pipelineResult.ValidatorRejected,
ErrorMessage: pipelineErr.Error(),
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
}
results = append(results, failed)
copy(workUnits[moduleIndex*len(sections):(moduleIndex+1)*len(sections)], pipelineResult.WorkUnits)
@@ -252,8 +228,6 @@ func (r *Runner) run(ctx context.Context, input RunInput) (RunOutput, error) {
ValidatorRejected: pipelineResult.ValidatorRejected,
AppliedChanges: applied,
SkippedChanges: skipped,
StartedAt: startedAt,
CompletedAt: time.Now().UTC(),
})
}
@@ -312,7 +286,6 @@ type collectSectionProposalsInput struct {
DiagnosticsDir string
ValidationClient contracts.StructuredLLMClient
ValidationDiagnosticsDir string
Timings *validatorTimingCollector
}
type sectionProposalResult struct {
@@ -515,7 +488,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
SectionEnriched: enriched,
ValidationLLMClient: input.ValidationClient,
DiagnosticsDir: input.ValidationDiagnosticsDir,
Timings: input.Timings,
ReportDiagnosticError: input.ReportDiagnosticError,
})
setErr(err)
@@ -599,7 +571,6 @@ type validateSectionCandidatesInput struct {
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
DiagnosticsDir string
Timings *validatorTimingCollector
}
type validateSectionCandidatesResult struct {
@@ -637,7 +608,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
}
validatorStarted := time.Now()
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
@@ -654,9 +624,6 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
DiagnosticsWriter: diagnosticsWriter,
ReportDiagnosticError: input.ReportDiagnosticError,
})
if input.Timings != nil {
input.Timings.record(input.Spec.ModuleKey, input.Spec.InstanceName, validator.Name(), validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked, time.Since(validatorStarted))
}
operations = append(operations, vResult.Operations...)
var decisionErr error

View File

@@ -753,9 +753,6 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
if len(out.ModuleResults[0].ValidatorDecisions) != 1 {
t.Fatalf("expected validator decisions recorded")
}
if len(out.ValidatorTimings) != 1 || out.ValidatorTimings[0].Calls != 1 || out.ValidatorTimings[0].ValidatorName != "allow" || out.RunWallTimeMS < 0 {
t.Fatalf("runner timings missing: %+v", out.ValidatorTimings)
}
}
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {

View File

@@ -1,56 +0,0 @@
package runner
import (
"sort"
"sync"
"time"
)
// ValidatorTiming records domain validator work without treating a scheduler
// permit or provider attempt as a logical LLM call.
type ValidatorTiming struct {
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ValidatorName string `json:"validator_name"`
LLMBacked bool `json:"llm_backed"`
Calls int `json:"calls"`
ElapsedMS int64 `json:"elapsed_ms"`
}
type validatorTimingCollector struct {
mu sync.Mutex
byKey map[string]*ValidatorTiming
}
func newValidatorTimingCollector() *validatorTimingCollector {
return &validatorTimingCollector{byKey: make(map[string]*ValidatorTiming)}
}
func (c *validatorTimingCollector) record(moduleKey, moduleInstance, validatorName string, llmBacked bool, elapsed time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
key := moduleInstance + "\x00" + validatorName
timing := c.byKey[key]
if timing == nil {
timing = &ValidatorTiming{ModuleKey: moduleKey, ModuleInstance: moduleInstance, ValidatorName: validatorName, LLMBacked: llmBacked}
c.byKey[key] = timing
}
timing.Calls++
timing.ElapsedMS += elapsed.Milliseconds()
}
func (c *validatorTimingCollector) snapshot() []ValidatorTiming {
c.mu.Lock()
defer c.mu.Unlock()
result := make([]ValidatorTiming, 0, len(c.byKey))
for _, timing := range c.byKey {
result = append(result, *timing)
}
sort.Slice(result, func(i, j int) bool {
if result[i].ModuleInstance != result[j].ModuleInstance {
return result[i].ModuleInstance < result[j].ModuleInstance
}
return result[i].ValidatorName < result[j].ValidatorName
})
return result
}