Publish domain artifacts without reporting sidecars

This commit is contained in:
2026-11-22 02:34:40 +00:00
parent 3b1b4c9bcf
commit 584b041e57
22 changed files with 241 additions and 1613 deletions

View File

@@ -7,7 +7,6 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkmap"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
@@ -127,27 +126,52 @@ func TestProfileValidationRejectsUnknownEvidenceLaneAndCopiesInputs(t *testing.T
}
}
func TestEncodeDeliverableSelectionAndArtifactCount(t *testing.T) {
chunkMap := acceptedChunkMapArtifact(t)
evidence := acceptedEvidenceContextArtifact(t)
request := contracts.OutputRequest{
NormalizeOutputs: []contracts.SerializedOutput{normalizeOutput("spells", `{"spell_casts":[]}`), normalizeOutput("notes/items", `{"items":[]}`)},
ChunkMap: &chunkMap, EvidenceContext: &evidence,
}
for _, test := range []struct {
name string
chunkMap, evidence bool
want []string
}{
{"lanes only", false, false, []string{"lanes/notes_items.json", "lanes/spells.json"}},
{"chunk map", true, false, []string{"chunk-map.json", "lanes/notes_items.json", "lanes/spells.json"}},
{"evidence", false, true, []string{"evidence-context.json", "lanes/notes_items.json", "lanes/spells.json"}},
{"both auxiliaries", true, true, []string{"chunk-map.json", "evidence-context.json", "lanes/notes_items.json", "lanes/spells.json"}},
} {
t.Run(test.name, func(t *testing.T) {
result, err := NewWithOptions(Options{IncludeChunkMap: test.chunkMap, EvidenceContext: pipeline.EvidenceContextPolicy{Enabled: test.evidence}}).Encode(context.Background(), request)
if err != nil {
t.Fatal(err)
}
if got := outputFileNames(result.Files); !reflect.DeepEqual(got, test.want) {
t.Fatalf("deliverable files=%v, want %v", got, test.want)
}
outcome := pipeline.RunOutput{OutputFiles: result.Files, FallbackCount: 1}
if outcome.ArtifactCount() != len(test.want) || outcome.FallbackCount != 1 {
t.Fatalf("outcome=%#v, want %d artifacts and one qualified fallback", outcome, len(test.want))
}
})
}
}
func TestEncodeEmptyOutputHasNoReportingSidecars(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{})
if err != nil || len(result.Files) != 0 {
t.Fatalf("empty output files=%v error=%v", outputFileNames(result.Files), err)
}
}
func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1", PipelineID: "pipeline-1"},
NormalizeOutputs: []contracts.SerializedOutput{
normalizeOutput("spells", `{"spell_casts":[{"spell":"Cure Wounds"}]}`),
normalizeOutput("notes/items", `{"items":[{"name":"Torch"}]}`),
normalizeOutput("notes/items", `{"items":[{"name":"Torch","metadata":{"context":"intrinsic domain data"}}]}`),
},
Rejected: []contracts.RejectedOutput{
{
Stage: "extract",
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-1",
ChunkIndex: 1,
ReasonCode: "invalid",
Message: "not accepted",
AttemptCount: 1,
ValidatorName: "validator",
},
},
Diagnostics: warningDiagnostics("warning", "check source"),
}
result, err := New().Encode(context.Background(), req)
@@ -156,13 +180,8 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
}
wantNames := []string{
"diagnostics.json",
"index.json",
"lanes/notes_items.json",
"lanes/spells.json",
"manifest.json",
"rejected.json",
"warnings.json",
}
if got := outputFileNames(result.Files); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("file names = %#v, want %#v", got, wantNames)
@@ -182,89 +201,16 @@ func TestEncodeReturnsLogicalFilesForNormalizedOutputs(t *testing.T) {
if spellCasts[0].(map[string]any)["spell"] != "Cure Wounds" {
t.Fatalf("spells output = %#v, want raw normalized content", spells)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
outputFiles := index["output_files"].([]any)
if len(outputFiles) != 2 {
t.Fatalf("len(index output_files) = %d, want 2", len(outputFiles))
}
firstIndex := outputFiles[0].(map[string]any)
secondIndex := outputFiles[1].(map[string]any)
if firstIndex["lane_id"] != "notes/items" || secondIndex["lane_id"] != "spells" {
t.Fatalf("output_files = %#v, want sorted by lane id", outputFiles)
}
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
if got := rejected["rejected"].([]any); len(got) != 1 {
t.Fatalf("rejected = %#v, want one rejected output", got)
}
}
func TestEncodeIncludesRejectedAndWarningsWhenEmpty(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
rejected := decodeObject(t, fileBytes(t, result.Files, "rejected.json"))
if got := rejected["rejected"].([]any); len(got) != 0 {
t.Fatalf("rejected = %#v, want empty array", got)
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
if got := warnings["groups"].([]any); len(got) != 0 {
t.Fatalf("warnings = %#v, want empty array", got)
}
}
func TestEncodePublishesVersionedDiagnosticPartitions(t *testing.T) {
diagnostics := contracts.DiagnosticCollection{
Groups: []contracts.DiagnosticGroup{
{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: "degraded",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk},
OccurrenceCount: 2,
Samples: []contracts.DiagnosticSample{{Scope: "chunk", Message: "degraded"}},
OmittedSampleCount: 1,
},
{
Disposition: contracts.DiagnosticDispositionAdvisory,
Category: contracts.DiagnosticCategoryDataQuality,
ReasonCode: "unrelated",
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageExtract},
OccurrenceCount: 3,
Samples: []contracts.DiagnosticSample{{Scope: "items[0]", Message: "unrelated"}},
OmittedSampleCount: 2,
},
},
Truncated: true,
UnrepresentedOccurrenceCount: 5,
}
result, err := New().Encode(context.Background(), contracts.OutputRequest{Diagnostics: diagnostics})
if err != nil {
t.Fatalf("Encode() error = %v", err)
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
if warnings["schema_version"] != warningsSchemaVersion || warnings["group_count"] != float64(1) || warnings["occurrence_count"] != float64(2) || len(warnings["groups"].([]any)) != 1 {
t.Fatalf("warnings file = %#v", warnings)
}
publishedDiagnostics := decodeObject(t, fileBytes(t, result.Files, "diagnostics.json"))
if publishedDiagnostics["schema_version"] != diagnosticsSchemaVersion || publishedDiagnostics["group_count"] != float64(1) || publishedDiagnostics["occurrence_count"] != float64(8) || publishedDiagnostics["truncated"] != true || publishedDiagnostics["unrepresented_occurrence_count"] != float64(5) || len(publishedDiagnostics["groups"].([]any)) != 1 {
t.Fatalf("diagnostics file = %#v", publishedDiagnostics)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if index["warnings_file"] != "warnings.json" || index["diagnostics_file"] != "diagnostics.json" {
t.Fatalf("index = %#v", index)
items := decodeObject(t, fileBytes(t, result.Files, "lanes/notes_items.json"))
metadata := items["items"].([]any)[0].(map[string]any)["metadata"].(map[string]any)
if metadata["context"] != "intrinsic domain data" {
t.Fatalf("intrinsic artifact metadata was lost: %#v", items)
}
}
func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
request := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
ChunkMap: &artifact,
}
for _, test := range []struct {
@@ -279,13 +225,9 @@ func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"diagnostics.json", "index.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
if got, want := outputFileNames(result.Files), []string{}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if _, ok := index["chunk_map"]; ok {
t.Fatalf("index = %#v, want no chunk map descriptor", index)
}
})
}
}
@@ -293,7 +235,6 @@ func TestEncodeChunkMapExportIsOptIn(t *testing.T) {
func TestEncodeIncludesValidatedChunkMap(t *testing.T) {
artifact := acceptedChunkMapArtifact(t)
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.SerializedOutput{
normalizeOutput("spells", `{"spell_casts":[]}`),
},
@@ -302,7 +243,7 @@ func TestEncodeIncludesValidatedChunkMap(t *testing.T) {
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "diagnostics.json", "index.json", "lanes/spells.json", "manifest.json", "rejected.json", "warnings.json"}; !reflect.DeepEqual(got, want) {
if got, want := outputFileNames(result.Files), []string{"chunk-map.json", "lanes/spells.json"}; !reflect.DeepEqual(got, want) {
t.Fatalf("file names = %#v, want %#v", got, want)
}
chunkMapFile := fileBytes(t, result.Files, chunkMapFileName)
@@ -312,22 +253,6 @@ func TestEncodeIncludesValidatedChunkMap(t *testing.T) {
if _, err := chunkmap.New().Decode(chunkMapFile); err != nil {
t.Fatalf("Decode(chunk map file) error = %v, want nil", err)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if got, want := index["chunk_map"], map[string]any{
"artifact_kind": "source/chunk-map",
"file": chunkMapFileName,
"media_type": chunkmap.MediaType,
"schema_id": chunkmap.SchemaID,
"schema_name": chunkmap.SchemaName,
"schema_version": chunkmap.SchemaVersion,
}; !reflect.DeepEqual(got, want) {
t.Fatalf("chunk map descriptor = %#v, want %#v", got, want)
}
for _, entry := range index["output_files"].([]any) {
if entry.(map[string]any)["file"] == chunkMapFileName {
t.Fatalf("output_files = %#v, want no chunk map", index["output_files"])
}
}
}
func TestEncodePreservesChunkMapAnnotationNumbers(t *testing.T) {
@@ -348,19 +273,13 @@ func TestEncodePreservesChunkMapAnnotationNumbers(t *testing.T) {
}
func TestEncodeOmitsChunkMapWithoutAcceptedArtifact(t *testing.T) {
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
})
result, err := NewWithOptions(Options{IncludeChunkMap: true}).Encode(context.Background(), contracts.OutputRequest{})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if got := outputFileNames(result.Files); containsString(got, chunkMapFileName) {
t.Fatalf("file names = %#v, want no chunk map", got)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if _, ok := index["chunk_map"]; ok {
t.Fatalf("index = %#v, want no chunk map descriptor", index)
}
}
func TestEncodeRejectsInvalidChunkMapArtifact(t *testing.T) {
@@ -387,7 +306,7 @@ func TestEncodeRejectsInvalidChunkMapArtifact(t *testing.T) {
func TestEncodeIncludesValidatedEvidenceContext(t *testing.T) {
artifact := acceptedEvidenceContextArtifact(t)
result, err := New().Encode(context.Background(), contracts.OutputRequest{EvidenceContext: &artifact})
result, err := NewWithOptions(Options{EvidenceContext: pipeline.EvidenceContextPolicy{Enabled: true}}).Encode(context.Background(), contracts.OutputRequest{EvidenceContext: &artifact})
if err != nil {
t.Fatalf("Encode() error = %v", err)
}
@@ -401,22 +320,6 @@ func TestEncodeIncludesValidatedEvidenceContext(t *testing.T) {
if len(value) != 1 || value[0].ID != 7 || value[0].Text != "Source content retained only in the evidence artifact." {
t.Fatalf("evidence context = %#v, want the published source-unit array", value)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if got, want := index["evidence_context"], map[string]any{
"artifact_kind": string(evidencecontext.ArtifactKind),
"file": evidenceContextFileName,
"media_type": evidencecontext.MediaType,
"schema_id": evidencecontext.SchemaID,
"schema_name": evidencecontext.SchemaName,
"schema_version": evidencecontext.SchemaVersion,
}; !reflect.DeepEqual(got, want) {
t.Fatalf("evidence context descriptor = %#v, want %#v", got, want)
}
for _, entry := range index["output_files"].([]any) {
if entry.(map[string]any)["file"] == evidenceContextFileName {
t.Fatalf("output_files = %#v, want no evidence context lane entry", index["output_files"])
}
}
}
func TestEncodeRejectsInvalidEvidenceContextArtifactWithoutContentLeakage(t *testing.T) {
@@ -436,7 +339,7 @@ func TestEncodeRejectsInvalidEvidenceContextArtifactWithoutContentLeakage(t *tes
t.Run(test.name, func(t *testing.T) {
candidate := contracts.CloneSerializedArtifact(artifact)
test.mutate(&candidate)
_, err := New().Encode(context.Background(), contracts.OutputRequest{EvidenceContext: &candidate})
_, err := NewWithOptions(Options{EvidenceContext: pipeline.EvidenceContextPolicy{Enabled: true}}).Encode(context.Background(), contracts.OutputRequest{EvidenceContext: &candidate})
if err == nil || err.Error() != "json output encoder: evidence context artifact is invalid" {
t.Fatalf("Encode() error = %v, want fixed evidence artifact error", err)
}
@@ -448,17 +351,13 @@ func TestEncodeRejectsInvalidEvidenceContextArtifactWithoutContentLeakage(t *tes
}
func TestEncodeOmitsEvidenceContextWhenArtifactIsAbsent(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{Manifest: artifacts.RunManifest{RunID: "run-1"}})
result, err := New().Encode(context.Background(), contracts.OutputRequest{})
if err != nil {
t.Fatalf("Encode() error = %v", err)
}
if got := outputFileNames(result.Files); containsString(got, evidenceContextFileName) {
t.Fatalf("file names = %#v, want no evidence context", got)
}
index := decodeObject(t, fileBytes(t, result.Files, "index.json"))
if _, ok := index["evidence_context"]; ok {
t.Fatalf("index = %#v, want no evidence context descriptor", index)
}
}
func TestEncodeChunkMapDoesNotMutateRequest(t *testing.T) {
@@ -482,145 +381,13 @@ func TestEncodeChunkMapDoesNotMutateRequest(t *testing.T) {
}
func TestEncodePrettyPrintsJSON(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
})
result, err := New().Encode(context.Background(), contracts.OutputRequest{NormalizeOutputs: []contracts.SerializedOutput{normalizeOutput("events", `{"value":1}`)}})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
t.Fatal(err)
}
manifest := string(fileBytes(t, result.Files, "manifest.json"))
if !strings.Contains(manifest, "\n \"run_id\": \"run-1\"\n") {
t.Fatalf("manifest JSON = %q, want two-space indentation", manifest)
}
}
func TestEncodePublishesLLMProfileProvenance(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
LLMProfiles: []artifacts.LLMProfileManifest{
{
ID: "endpoint-profile",
Provider: "promptkit",
Model: "endpoint-model",
ReasoningEffort: "low",
},
{
ID: "profile",
Provider: "promptkit",
Model: "model",
BackendID: "openrouter",
ReasoningEffort: "high",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
profiles := manifest["llm_profiles"].([]any)
if len(profiles) != 2 {
t.Fatalf("llm_profiles = %#v, want two entries", profiles)
}
endpointProfile := profiles[0].(map[string]any)
if _, exists := endpointProfile["backend_id"]; exists || endpointProfile["reasoning_effort"] != "low" {
t.Fatalf("endpoint-only LLM profile = %#v, want omitted backend and published reasoning", endpointProfile)
}
backendProfile := profiles[1].(map[string]any)
if backendProfile["backend_id"] != "openrouter" || backendProfile["reasoning_effort"] != "high" {
t.Fatalf("backend LLM profile = %#v, want published backend and reasoning fields", backendProfile)
}
}
func TestEncodeIncludesManifestReferences(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
References: []artifacts.ReferenceProvenance{
{
Stage: "extract",
LaneID: "events",
SlotName: "roster",
OriginType: "file",
OriginURI: "file:///tmp/roster.txt",
Digest: "sha256:reference",
MediaType: "text/plain; charset=utf-8",
SizeBytes: 12,
BindingSource: "config",
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
references := manifest["references"].([]any)
if len(references) != 1 {
t.Fatalf("references = %#v, want one entry", references)
}
reference := references[0].(map[string]any)
if reference["lane_id"] != "events" || reference["slot_name"] != "roster" || reference["digest"] != "sha256:reference" {
t.Fatalf("reference manifest = %#v, want lane slot digest", reference)
}
if _, ok := reference["content"]; ok {
t.Fatalf("reference manifest = %#v, want no content field", reference)
}
}
func TestEncodeIncludesManifestRawOutputProvenance(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
Manifest: artifacts.RunManifest{
RunID: "run-1",
NormalizedOutputs: []artifacts.NormalizedOutputManifest{
{
LaneID: "spells",
ModuleKey: "noop",
SourceID: "source-1",
MediaType: contentTypeJSON,
Schema: artifacts.OutputSchemaProvenance{
ID: "schema-id",
Name: "schema-name",
Version: "v1",
},
},
},
RejectedOutputs: []artifacts.RejectedOutputManifest{
{
Stage: "extract",
LaneID: "spells",
ModuleKey: "dnd/spells",
ChunkID: "chunk-0",
ReasonCode: "raw_output_rejected",
AttemptCount: 2,
},
},
},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
manifest := decodeObject(t, fileBytes(t, result.Files, "manifest.json"))
normalized := manifest["normalized_outputs"].([]any)
if len(normalized) != 1 {
t.Fatalf("normalized_outputs = %#v, want one entry", normalized)
}
normalizedEntry := normalized[0].(map[string]any)
if normalizedEntry["lane_id"] != "spells" || normalizedEntry["media_type"] != contentTypeJSON {
t.Fatalf("normalized output manifest = %#v, want lane and media type", normalizedEntry)
}
rejected := manifest["rejected_outputs"].([]any)
if len(rejected) != 1 {
t.Fatalf("rejected_outputs = %#v, want one entry", rejected)
}
rejectedEntry := rejected[0].(map[string]any)
if rejectedEntry["attempt_count"] != float64(2) || rejectedEntry["chunk_id"] != "chunk-0" {
t.Fatalf("rejected output manifest = %#v, want attempt count and chunk", rejectedEntry)
content := string(fileBytes(t, result.Files, "lanes/events.json"))
if !strings.Contains(content, "\n \"value\": 1\n") {
t.Fatalf("artifact JSON=%q, want two-space indentation", content)
}
}
@@ -703,14 +470,9 @@ func TestEncodeRejectsSanitizedFilenameCollisions(t *testing.T) {
func TestEncodeDoesNotMutateInputs(t *testing.T) {
req := contracts.OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []contracts.SerializedOutput{
normalizeOutput("spells", `{"name":"original"}`),
},
Rejected: []contracts.RejectedOutput{
{Stage: "extract", LaneID: "spells", Message: "not accepted"},
},
Diagnostics: warningDiagnostics("warning", "message"),
}
before := mustMarshal(t, req)
@@ -725,43 +487,10 @@ func TestEncodeDoesNotMutateInputs(t *testing.T) {
req.NormalizeOutputs[0].Artifact.Content[0] = '['
req.NormalizeOutputs[0].Artifact.Metadata["name"] = "changed"
req.Rejected[0].Message = "changed"
req.Diagnostics.Groups[0].Samples[0].Message = "changed"
if !stdjson.Valid(fileBytes(t, result.Files, "lanes/spells.json")) {
t.Fatal("output changed after request mutation")
}
warnings := decodeObject(t, fileBytes(t, result.Files, "warnings.json"))
gotWarnings := warnings["groups"].([]any)
if gotWarnings[0].(map[string]any)["samples"].([]any)[0].(map[string]any)["message"] != "message" {
t.Fatalf("warnings output changed after request mutation: %#v", gotWarnings)
}
}
func TestOutputFilesDoNotContainWarnings(t *testing.T) {
result, err := New().Encode(context.Background(), contracts.OutputRequest{
NormalizeOutputs: []contracts.SerializedOutput{normalizeOutput("spells", `{"spell":"Shield"}`)},
Diagnostics: warningDiagnostics("pipeline-warning", "warning"),
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
outputFile := decodeObject(t, fileBytes(t, result.Files, "lanes/spells.json"))
if _, ok := outputFile["warnings"]; ok {
t.Fatalf("output file contains warnings: %#v", outputFile)
}
}
func warningDiagnostics(reasonCode, message string) contracts.DiagnosticCollection {
return contracts.DiagnosticCollection{Groups: []contracts.DiagnosticGroup{{
Disposition: contracts.DiagnosticDispositionWarning,
Category: contracts.DiagnosticCategoryDegradation,
ReasonCode: reasonCode,
Origin: contracts.DiagnosticOrigin{Stage: contracts.DiagnosticOriginStageChunk},
OccurrenceCount: 1,
Samples: []contracts.DiagnosticSample{{Scope: "test", Message: message}},
}}}
}
func normalizeOutput(laneID string, content string) contracts.SerializedOutput {