Remove retired reporting and diagnostic code
This commit is contained in:
@@ -5,9 +5,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
@@ -21,81 +19,6 @@ func mapLLMSummary(source llm.ExecutionSummary) reporting.LLMSummary {
|
||||
}
|
||||
}
|
||||
|
||||
// BuildUtilization groups complete logical calls by their selected backend.
|
||||
func BuildUtilization(output *runner.RunOutput, backends map[string]*pk.BackendInspection) reporting.UtilizationDiagnostics {
|
||||
result := reporting.UtilizationDiagnostics{Backends: make(map[string]reporting.BackendUtilization, len(backends))}
|
||||
if output == nil {
|
||||
return result
|
||||
}
|
||||
result.RunWallTimeMS = output.RunWallTimeMS
|
||||
groups := make(map[string]*llm.ObservationSnapshot, len(backends))
|
||||
callBackends := make(map[uint64]string, len(output.Observations.Calls))
|
||||
moduleIndices := make(map[string]int)
|
||||
moduleIndex := func(key, instance string) int {
|
||||
if i, ok := moduleIndices[instance]; ok {
|
||||
return i
|
||||
}
|
||||
i := len(result.Modules)
|
||||
moduleIndices[instance] = i
|
||||
result.Modules = append(result.Modules, reporting.ModuleUtilization{ModuleKey: key, ModuleInstance: instance})
|
||||
return i
|
||||
}
|
||||
// Seed configured order, including modules that made no LLM calls.
|
||||
for _, module := range output.ModuleResults {
|
||||
i := moduleIndex(module.ModuleKey, module.ModuleInstance)
|
||||
result.Modules[i].ModuleWallTimeMS = module.CompletedAt.Sub(module.StartedAt).Milliseconds()
|
||||
}
|
||||
for _, call := range output.Observations.Calls {
|
||||
id := call.Prompt.BackendID
|
||||
if groups[id] == nil {
|
||||
groups[id] = &llm.ObservationSnapshot{}
|
||||
}
|
||||
groups[id].Calls = append(groups[id].Calls, call)
|
||||
callBackends[call.CallID] = id
|
||||
if call.Context.ModuleInstance != "" {
|
||||
i := moduleIndex(call.Context.ModuleKey, call.Context.ModuleInstance)
|
||||
switch call.Context.Role {
|
||||
case "proposal":
|
||||
result.Modules[i].ProposalLLMCalls++
|
||||
case "validation":
|
||||
result.Modules[i].ValidationLLMCalls++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, event := range output.Observations.Events {
|
||||
if id, ok := callBackends[event.CallID]; ok {
|
||||
groups[id].Events = append(groups[id].Events, event)
|
||||
}
|
||||
}
|
||||
for id, backend := range backends {
|
||||
group := groups[id]
|
||||
if group == nil {
|
||||
group = &llm.ObservationSnapshot{}
|
||||
}
|
||||
var wait, generation, retry, wall time.Duration
|
||||
for _, call := range group.Calls {
|
||||
wall += call.WallDuration
|
||||
}
|
||||
for _, event := range group.Events {
|
||||
if event.Kind == string(pk.ExecutionFinished) {
|
||||
wait += event.CapacityWait
|
||||
generation += event.GenerationDuration
|
||||
retry += event.RetryDelay
|
||||
}
|
||||
}
|
||||
result.Backends[id] = reporting.BackendUtilization{
|
||||
Concurrency: backend.ConcurrencyLimit, QueueCapacity: backend.QueueCapacity,
|
||||
LLMSummary: mapLLMSummary(llm.Summarize(*group)),
|
||||
CapacityWaitMS: wait.Milliseconds(), GenerationMS: generation.Milliseconds(),
|
||||
RetryDelayMS: retry.Milliseconds(), LogicalWallMS: wall.Milliseconds(),
|
||||
}
|
||||
}
|
||||
for _, timing := range output.ValidatorTimings {
|
||||
result.Validators = append(result.Validators, reporting.ValidatorUtilization{ModuleKey: timing.ModuleKey, ModuleInstance: timing.ModuleInstance, ValidatorKey: timing.ValidatorName, LLMBacked: timing.LLMBacked, Calls: timing.Calls, ElapsedMS: timing.ElapsedMS})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type executionCall struct {
|
||||
CallID uint64 `json:"call_id"`
|
||||
Context structuredoutput.DiagnosticContext `json:"context"`
|
||||
|
||||
@@ -8,12 +8,11 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
|
||||
func TestExecutionReportingKeepsSafeMilliseconds(t *testing.T) {
|
||||
start := time.Now().UTC()
|
||||
snapshot := llm.ObservationSnapshot{
|
||||
Calls: []llm.LogicalCall{
|
||||
@@ -22,14 +21,6 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
|
||||
},
|
||||
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
|
||||
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"primary": {ConcurrencyLimit: intPointer(2), QueueCapacity: intPointer(3)}, "secondary": {ConcurrencyLimit: intPointer(1)}})
|
||||
if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
|
||||
t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
|
||||
}
|
||||
if utilization.Backends["secondary"].LogicalCalls != 1 || utilization.Backends["secondary"].ProviderAttempts != 0 || !utilization.Backends["secondary"].Usage.Total.Known {
|
||||
t.Fatalf("predispatch backend: %+v", utilization.Backends["secondary"])
|
||||
}
|
||||
raw, err := json.Marshal(BuildLLMExecution(snapshot))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -57,17 +48,11 @@ func BenchmarkExecutionReports(b *testing.B) {
|
||||
snapshot.Calls = append(snapshot.Calls, llm.LogicalCall{CallID: uint64(i + 1), Prompt: structuredoutput.PromptMetadata{BackendID: "local"}, Context: structuredoutput.DiagnosticContext{ModuleInstance: "grammar", Role: "proposal"}})
|
||||
snapshot.Events = append(snapshot.Events, llm.ExecutionObservation{CallID: uint64(i + 1), Kind: string(pk.ExecutionFinished), Attempts: 1})
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot}
|
||||
b.Run(fmt.Sprintf("execution/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
BuildLLMExecution(snapshot)
|
||||
}
|
||||
})
|
||||
b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
BuildUtilization(output, map[string]*pk.BackendInspection{"local": {ConcurrencyLimit: intPointer(1)}})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,15 +93,4 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
|
||||
if execution.Calls[1].Summary.Usage.Total.Value != 7 || !execution.Calls[1].Summary.Usage.Total.Complete || execution.Calls[0].Summary.Usage.Total.Known {
|
||||
t.Fatalf("call usage: %+v", execution.Calls)
|
||||
}
|
||||
output := &runner.RunOutput{Observations: snapshot}
|
||||
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"a": {}, "b": {}})
|
||||
total := utilization.Backends["a"].Usage.Total
|
||||
if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
|
||||
t.Fatalf("partial backend usage: %+v", utilization)
|
||||
}
|
||||
if len(utilization.Modules) != 2 || utilization.Modules[1].ProposalLLMCalls != 1 || utilization.Modules[1].ValidationLLMCalls != 1 {
|
||||
t.Fatalf("module grouping: %+v", utilization.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
func intPointer(value int) *int { return &value }
|
||||
|
||||
@@ -25,7 +25,6 @@ type BuildInput struct {
|
||||
StartedAt time.Time
|
||||
CompletedAt *time.Time
|
||||
TranscriptPath string
|
||||
GlossaryPath string
|
||||
OutputPath string
|
||||
ModuleSpecs []contracts.ModuleRunSpec
|
||||
RunOutput *runner.RunOutput
|
||||
@@ -36,7 +35,7 @@ type BuildInput struct {
|
||||
func Build(input BuildInput) reporting.Snapshot {
|
||||
snapshot := reporting.Snapshot{
|
||||
RunID: input.RunID, Status: input.Status, StartedAt: input.StartedAt.UTC(), CompletedAt: input.CompletedAt,
|
||||
TranscriptPath: input.TranscriptPath, GlossaryPath: input.GlossaryPath, OutputPath: input.OutputPath,
|
||||
TranscriptPath: input.TranscriptPath, OutputPath: input.OutputPath,
|
||||
StageNames: make([]string, len(input.ModuleSpecs)), Errors: make([]reporting.ErrorRecord, 0),
|
||||
Recoveries: make([]reporting.RecoveryRecord, 0), WorkUnits: make([]reporting.WorkUnit, 0),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user