From dd25838589465982a71f659f86099d6d9314291b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 09:38:24 +0000 Subject: [PATCH] Keep long fallback reasons from invalidating warnings --- docs/roadmap/remediation.md | 4 ++ internal/cli/fallback_diagnostic_test.go | 69 +++++++++++++++++++ internal/framework/pipeline/diagnostics.go | 16 +++++ .../pipeline/fallback_diagnostic_test.go | 37 ++++++++++ internal/framework/pipeline/runner_cache.go | 10 +-- .../framework/pipeline/runner_operation.go | 2 +- 6 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 internal/cli/fallback_diagnostic_test.go create mode 100644 internal/framework/pipeline/fallback_diagnostic_test.go diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index ce7fe7ca..d82e2091 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -235,6 +235,10 @@ do not invent a generic identity framework for families that do not need one. ## Finding 4: Bound Framework-Generated Fallback Diagnostic Messages +**Status: Complete.** Framework fallback/cache samples share bounded UTF-8 +rendering without changing full outcome evidence or module diagnostic checks. +Multibyte/invalid-UTF-8 regressions and a typed spell CLI fallback probe pass. + Priority: P2. Relevant stages: 9, 11, 24, 29, and 34. ### Evidence And Impact diff --git a/internal/cli/fallback_diagnostic_test.go b/internal/cli/fallback_diagnostic_test.go new file mode 100644 index 00000000..2f875707 --- /dev/null +++ b/internal/cli/fallback_diagnostic_test.go @@ -0,0 +1,69 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "go.maximumdirect.net/promptkit" +) + +type longProviderFailure struct{} + +func (longProviderFailure) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + return nil, fmt.Errorf("provider failed: %s", strings.Repeat("a detailed upstream failure; ", 250)) +} + +func TestPermissiveFallbackSurvivesLongNativeCause(t *testing.T) { + directory := t.TempDir() + configPath, inputPath := filepath.Join(directory, "config.yml"), filepath.Join(directory, "transcript.json") + configuration := fmt.Sprintf(`version: 6 +output: {directory: %q} +cache_directory: %q +promptkit: + profiles: + - id: audit + endpoint: https://example.invalid/v1 + model: audit + api_key_required: false +pipelines: + audit: + llm_profile: audit + mode: permissive + max_additional_turns: 0 + input: {module: seriatim} + artifacts: + spells: + extract: {module: dnd/spells} + normalize: {module: dnd/spells} +`, filepath.Join(directory, "output"), filepath.Join(directory, "cache")) + if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(inputPath, []byte(`{"metadata":{"id":"audit"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`), 0o600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + status := RunWithOptions([]string{"run", "audit", "--config", configPath, "--input-file", inputPath, "--cache", "bypass", "--llm-transient-retries=0", "--format", "json", "--progress=false"}, &stdout, &stderr, Options{LLMClient: longProviderFailure{}}) + if status != 0 { + t.Fatalf("supported empty fallback failed: status=%d stderr=%s", status, stderr.String()) + } + var report struct { + FallbackCount int `json:"fallback_count"` + Directory string `json:"output_directory"` + } + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.FallbackCount != 1 { + t.Fatalf("fallback count = %d", report.FallbackCount) + } + if _, err := os.Stat(filepath.Join(report.Directory, "lanes", "spells.json")); err != nil { + t.Fatal(err) + } +} diff --git a/internal/framework/pipeline/diagnostics.go b/internal/framework/pipeline/diagnostics.go index 03564250..28921fc7 100644 --- a/internal/framework/pipeline/diagnostics.go +++ b/internal/framework/pipeline/diagnostics.go @@ -1,11 +1,27 @@ package pipeline import ( + "strings" + "unicode/utf8" + "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" frameworkdiagnostics "gitea.maximumdirect.net/eric/notarius/internal/framework/diagnostics" ) +// boundedDiagnosticMessage is for framework-authored samples only. Module +// diagnostics still pass through ordinary contract validation unchanged. +func boundedDiagnosticMessage(message string) string { + message = strings.ToValidUTF8(message, "�") + if len(message) > contracts.MaxDiagnosticMessageBytes { + message = message[:contracts.MaxDiagnosticMessageBytes] + for !utf8.ValidString(message) { + message = message[:len(message)-1] + } + } + return message +} + func promoteProducerDiagnostics(diagnostics []contracts.ProducerDiagnostic, origin contracts.DiagnosticOrigin, chunk *source.Chunk) ([]contracts.DiagnosticGroup, error) { if err := contracts.ValidateProducerDiagnostics(diagnostics); err != nil { return nil, err diff --git a/internal/framework/pipeline/fallback_diagnostic_test.go b/internal/framework/pipeline/fallback_diagnostic_test.go new file mode 100644 index 00000000..e0b723d8 --- /dev/null +++ b/internal/framework/pipeline/fallback_diagnostic_test.go @@ -0,0 +1,37 @@ +package pipeline + +import ( + "strings" + "testing" + "unicode/utf8" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" +) + +func TestFallbackDiagnosticBoundsOnlyThePresentedReason(t *testing.T) { + for _, reason := range []string{ + strings.Repeat("a", contracts.MaxDiagnosticMessageBytes-1) + "界界", + "provider error: " + string([]byte{0xff}) + strings.Repeat("x", contracts.MaxDiagnosticMessageBytes), + } { + outcome := contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: reason} + groups, err := operationDiagnosticGroups(nil, nil, contracts.CandidateAssessment{}, outcome, contracts.DiagnosticOrigin{Stage: "extract", ModuleKey: "test"}, nil) + if err != nil { + t.Fatal(err) + } + if len(groups) != 1 || len(groups[0].Samples) != 1 { + t.Fatalf("missing fallback warning: %#v", groups) + } + message := groups[0].Samples[0].Message + if strings.TrimSpace(message) == "" || len(message) > contracts.MaxDiagnosticMessageBytes || !utf8.ValidString(message) { + t.Fatalf("invalid warning sample: %q", message) + } + if outcome.FallbackReason != reason { + t.Fatal("original failure reason was changed") + } + invalid := []contracts.ProducerDiagnostic{{Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, + ReasonCode: "module_warning", OccurrenceCount: 1, Samples: []contracts.DiagnosticSample{{Scope: "test", Message: reason}}}} + if _, err := operationDiagnosticGroups(invalid, nil, contracts.CandidateAssessment{}, outcome, contracts.DiagnosticOrigin{Stage: "extract", ModuleKey: "test"}, nil); err == nil { + t.Fatal("invalid module diagnostic was silently repaired") + } + } +} diff --git a/internal/framework/pipeline/runner_cache.go b/internal/framework/pipeline/runner_cache.go index 511267a3..fc2b38f3 100644 --- a/internal/framework/pipeline/runner_cache.go +++ b/internal/framework/pipeline/runner_cache.go @@ -3,8 +3,6 @@ package pipeline import ( "context" "fmt" - "strings" - "unicode/utf8" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" @@ -47,13 +45,7 @@ func producerCacheObserver(input RunInput, origin contracts.DiagnosticOrigin, ch observation.ChunkID, observation.ChunkIndex = chunk.ID, &chunkIndex } if observation.Err != nil { - message := strings.ToValidUTF8(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err), "�") - if len(message) > contracts.MaxDiagnosticMessageBytes { - message = message[:contracts.MaxDiagnosticMessageBytes] - for !utf8.ValidString(message) { - message = message[:len(message)-1] - } - } + message := boundedDiagnosticMessage(fmt.Sprintf("Artifact cache %s: %v", observation.Decision, observation.Err)) *warnings = append(*warnings, contracts.ProducerDiagnostic{ Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryDegradation, ReasonCode: "cache_" + observation.Decision, OccurrenceCount: 1, diff --git a/internal/framework/pipeline/runner_operation.go b/internal/framework/pipeline/runner_operation.go index eece7101..7d643a51 100644 --- a/internal/framework/pipeline/runner_operation.go +++ b/internal/framework/pipeline/runner_operation.go @@ -52,7 +52,7 @@ func operationDiagnosticGroups(diagnostics, cacheDiagnostics []contracts.Produce warning := contracts.ProducerDiagnostic{ Disposition: contracts.DiagnosticDispositionWarning, Category: contracts.DiagnosticCategoryFallback, ReasonCode: "producer_fallback", OccurrenceCount: 1, - Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: outcome.FallbackReason}}, + Samples: []contracts.DiagnosticSample{{Scope: origin.ModuleKey, Message: boundedDiagnosticMessage(outcome.FallbackReason)}}, } fallbackGroups, err := promoteProducerDiagnostics([]contracts.ProducerDiagnostic{warning}, origin, chunk) return append(groups, fallbackGroups...), err