notarius/internal/framework/chunkplan/store_test.go

package chunkplan

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)

const testSourceDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

func TestFilesystemStoreRoundTripAndExactPath(t *testing.T) {
	root := filepath.Join(t.TempDir(), "plans")
	store := newStore(t, root)
	record := testRecord(t, 1)
	if err := store.Save(record); err != nil {
		t.Fatalf("Save() error = %v", err)
	}

	target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
	data, err := os.ReadFile(target)
	if err != nil {
		t.Fatalf("read exact plan path: %v", err)
	}
	if bytes.Contains(data, []byte("RAW REFERENCE CONTENT")) {
		t.Fatal("stored record contains raw reference content")
	}
	var envelope struct {
		Producer struct {
			References []map[string]any `json:"references"`
		} `json:"producer"`
	}
	if err := json.Unmarshal(data, &envelope); err != nil {
		t.Fatal(err)
	}
	if len(envelope.Producer.References) != 1 {
		t.Fatalf("stored references = %#v", envelope.Producer.References)
	}
	if _, exists := envelope.Producer.References[0]["content"]; exists {
		t.Fatalf("stored reference contains content field: %#v", envelope.Producer.References[0])
	}

	got, decision, err := store.Load(testSourceDigest)
	if err != nil || decision.Status != pipeline.ChunkPlanHit {
		t.Fatalf("Load() decision=%#v error=%v", decision, err)
	}
	if !reflect.DeepEqual(got, record) {
		t.Fatalf("round trip record = %#v, want %#v", got, record)
	}
}

func TestFilesystemStoreMissingAndOperationalErrors(t *testing.T) {
	root := filepath.Join(t.TempDir(), "plans")
	store := newStore(t, root)
	_, decision, err := store.Load(testSourceDigest)
	if err != nil || decision.Status != pipeline.ChunkPlanMissing {
		t.Fatalf("missing decision=%#v error=%v", decision, err)
	}
	if _, err := os.Stat(root); !os.IsNotExist(err) {
		t.Fatalf("Load() created missing root: %v", err)
	}

	target := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
	if err := os.MkdirAll(target, 0o700); err != nil {
		t.Fatal(err)
	}
	_, decision, err = store.Load(testSourceDigest)
	if err != nil || decision.Status != pipeline.ChunkPlanInvalid {
		t.Fatalf("Load(plan.json directory) decision=%#v error=%v", decision, err)
	}
}

func TestFilesystemStoreRejectsSymlinkedEntries(t *testing.T) {
	tests := []struct {
		name  string
		setup func(t *testing.T, root, outside string)
	}{
		{
			name: "digest directory",
			setup: func(t *testing.T, root, outside string) {
				t.Helper()
				if err := os.MkdirAll(outside, 0o700); err != nil {
					t.Fatal(err)
				}
				writeFile(t, filepath.Join(outside, "plan.json"), []byte("outside plan"), 0o640)
				createSymlink(t, outside, filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:")))
			},
		},
		{
			name: "plan file",
			setup: func(t *testing.T, root, outside string) {
				t.Helper()
				digestDir := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"))
				if err := os.MkdirAll(digestDir, 0o700); err != nil {
					t.Fatal(err)
				}
				writeFile(t, outside, []byte("outside plan"), 0o640)
				createSymlink(t, outside, filepath.Join(digestDir, "plan.json"))
			},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			root := filepath.Join(t.TempDir(), "plans")
			if err := os.MkdirAll(root, 0o700); err != nil {
				t.Fatal(err)
			}
			outside := filepath.Join(t.TempDir(), "outside")
			tc.setup(t, root, outside)
			before := readFile(t, outsidePlanPath(tc.name, outside))
			beforeMode := fileMode(t, outsidePlanPath(tc.name, outside))

			store := newStore(t, root)
			_, decision, err := store.Load(testSourceDigest)
			if err != nil || decision.Status != pipeline.ChunkPlanInvalid {
				t.Fatalf("Load() decision=%#v error=%v", decision, err)
			}
			if err := store.Save(testRecord(t, 1)); err == nil {
				t.Fatal("Save() error = nil")
			}
			outsidePlan := outsidePlanPath(tc.name, outside)
			if got := readFile(t, outsidePlan); !bytes.Equal(got, before) {
				t.Fatalf("outside content = %q, want %q", got, before)
			}
			if got := fileMode(t, outsidePlan); got != beforeMode {
				t.Fatalf("outside mode = %04o, want %04o", got, beforeMode)
			}
		})
	}
}

func TestOpenDigestRootRejectsEntryReplacedDuringOpen(t *testing.T) {
	rootPath := t.TempDir()
	digestDir := strings.TrimPrefix(testSourceDigest, "sha256:")
	if err := os.Mkdir(filepath.Join(rootPath, digestDir), 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.Mkdir(filepath.Join(rootPath, "redirect"), 0o700); err != nil {
		t.Fatal(err)
	}
	root, err := os.OpenRoot(rootPath)
	if err != nil {
		t.Fatal(err)
	}
	defer root.Close()

	traced := false
	opened, state, err := openDigestRootWithHooks(root, digestDir, false, digestOpenHooks{BeforeOpen: func() error {
		if traced {
			return nil
		}
		traced = true
		if err := os.Rename(filepath.Join(rootPath, digestDir), filepath.Join(rootPath, "original")); err != nil {
			return err
		}
		return os.Symlink("redirect", filepath.Join(rootPath, digestDir))
	}})
	if opened != nil {
		_ = opened.Close()
	}
	if err != nil || state != entryRejected {
		t.Fatalf("openDigestRootWithHooks() root=%v state=%v error=%v, want nil/rejected/nil", opened, state, err)
	}
}

func TestReadPlanRejectsEntryReplacedDuringOpen(t *testing.T) {
	rootPath := t.TempDir()
	writeFile(t, filepath.Join(rootPath, planFileName), []byte("original"), 0o600)
	writeFile(t, filepath.Join(rootPath, "redirect.json"), []byte("redirect"), 0o600)
	root, err := os.OpenRoot(rootPath)
	if err != nil {
		t.Fatal(err)
	}
	defer root.Close()

	data, state, err := readPlanWithHooks(root, planReadHooks{BeforeOpen: func() error {
		if err := os.Remove(filepath.Join(rootPath, planFileName)); err != nil {
			return err
		}
		return os.Symlink("redirect.json", filepath.Join(rootPath, planFileName))
	}})
	if err != nil || state != entryRejected || data != nil {
		t.Fatalf("readPlanWithHooks() data=%q state=%v error=%v, want nil/rejected/nil", data, state, err)
	}
}

func TestFilesystemStoreRejectsUnexpectedEntryTypes(t *testing.T) {
	tests := []struct {
		name  string
		setup func(t *testing.T, root string)
	}{
		{
			name: "digest file",
			setup: func(t *testing.T, root string) {
				writeFile(t, filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:")), []byte("not a directory"), 0o600)
			},
		},
		{
			name: "plan directory",
			setup: func(t *testing.T, root string) {
				if err := os.MkdirAll(filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json"), 0o700); err != nil {
					t.Fatal(err)
				}
			},
		},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			root := t.TempDir()
			tc.setup(t, root)
			store := newStore(t, root)
			_, decision, err := store.Load(testSourceDigest)
			if err != nil || decision.Status != pipeline.ChunkPlanInvalid {
				t.Fatalf("Load() decision=%#v error=%v", decision, err)
			}
			if err := store.Save(testRecord(t, 1)); err == nil {
				t.Fatal("Save() error = nil")
			}
		})
	}
}

func TestFilesystemStoreRejectsMalformedSourceDigests(t *testing.T) {
	store := newStore(t, t.TempDir())
	for _, digest := range []string{"", "sha1:" + strings.Repeat("a", 64), "sha256:../escape", "sha256:" + strings.Repeat("A", 64), "sha256:" + strings.Repeat("a", 63)} {
		t.Run(digest, func(t *testing.T) {
			if _, _, err := store.Load(digest); err == nil {
				t.Fatal("Load() error = nil")
			}
			record := testRecord(t, 1)
			record.SourceDigest = digest
			record.Plan.SourceDigest = digest
			record.PlanDigest, _ = source.DigestChunkPlan(record.Plan)
			if err := store.Save(record); err == nil {
				t.Fatal("Save() error = nil")
			}
		})
	}
}

func TestFilesystemStoreReportsInvalidRecordsAsRecoverable(t *testing.T) {
	tests := []struct {
		name   string
		mutate func([]byte) []byte
	}{
		{name: "unknown field", mutate: func(data []byte) []byte {
			return bytes.Replace(data, []byte(`{"schema_version"`), []byte(`{"SENTINEL_UNKNOWN_FIELD":true,"schema_version"`), 1)
		}},
		{name: "truncated JSON", mutate: func(data []byte) []byte { return data[:len(data)/2] }},
		{name: "legacy v2 record", mutate: replaceJSON(`notarius.chunk-plan.v3`, `notarius.chunk-plan.v2`)},
		{name: "source mismatch", mutate: replaceJSON(testSourceDigest, "sha256:"+strings.Repeat("b", 64))},
		{name: "plan digest mismatch", mutate: func(data []byte) []byte {
			prefix := []byte(`"plan_digest":"sha256:`)
			index := bytes.Index(data, prefix)
			if index >= 0 {
				data[index+len(prefix)] = '0'
			}
			return data
		}},
		{name: "noncanonical annotation", mutate: func(data []byte) []byte {
			return bytes.Replace(data, []byte(`"test/value"`), []byte(`"SENTINEL_ANNOTATION_NAMESPACE"`), 1)
		}},
		{name: "bad boundary", mutate: func(data []byte) []byte {
			return bytes.Replace(data, []byte(`"start_unit_id":1`), []byte(`"start_unit_id":0`), 1)
		}},
		{name: "timestamp", mutate: replaceJSON(`2026-07-18T12:00:00Z`, `SENTINEL_TIMESTAMP`)},
		{name: "trailing JSON", mutate: func(data []byte) []byte { return append(data, []byte(` {"SENTINEL_TRAILING":true}`)...) }},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			root := t.TempDir()
			store := newStore(t, root)
			record := testRecord(t, 1)
			if err := store.Save(record); err != nil {
				t.Fatal(err)
			}
			path := filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:"), "plan.json")
			data, err := os.ReadFile(path)
			if err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(path, tc.mutate(data), 0o600); err != nil {
				t.Fatal(err)
			}
			got, decision, err := store.Load(testSourceDigest)
			if err != nil || decision.Status != pipeline.ChunkPlanInvalid || !reflect.DeepEqual(got, pipeline.ChunkPlanRecord{}) || decision.Reason != "stored chunk plan is invalid" {
				t.Fatalf("record=%#v decision=%#v error=%v", got, decision, err)
			}
			for _, sentinel := range []string{"SENTINEL_UNKNOWN_FIELD", "SENTINEL_SCHEMA_VALUE", "SENTINEL_ANNOTATION_NAMESPACE", "SENTINEL_TIMESTAMP", "SENTINEL_TRAILING"} {
				if strings.Contains(decision.Reason, sentinel) {
					t.Fatalf("decision leaked %q: %#v", sentinel, decision)
				}
			}
		})
	}
}

func TestFilesystemStoreAtomicallyReplacesAndPreservesValidRecordOnFailure(t *testing.T) {
	root := t.TempDir()
	store := newStore(t, root)
	first := testRecord(t, 1)
	second := testRecord(t, 2)
	if err := store.Save(first); err != nil {
		t.Fatal(err)
	}
	invalid := second
	invalid.PlanDigest = "sha256:" + strings.Repeat("0", 64)
	if err := store.Save(invalid); err == nil {
		t.Fatal("Save(invalid) error = nil")
	}
	got, _, _ := store.Load(testSourceDigest)
	if !reflect.DeepEqual(got, first) {
		t.Fatalf("record after failed replacement = %#v", got)
	}
	if err := store.Save(second); err != nil {
		t.Fatal(err)
	}
	got, _, _ = store.Load(testSourceDigest)
	if !reflect.DeepEqual(got, second) {
		t.Fatalf("record after replacement = %#v", got)
	}
	assertNoTemps(t, filepath.Join(root, strings.TrimPrefix(testSourceDigest, "sha256:")))
}

func TestFilesystemStoreConcurrentWritersExposeCompleteRecord(t *testing.T) {
	store := newStore(t, t.TempDir())
	const writers = 24
	records := make([]pipeline.ChunkPlanRecord, writers)
	for i := range records {
		records[i] = testRecord(t, i+1)
	}
	var wg sync.WaitGroup
	errs := make(chan error, writers)
	for i := range records {
		wg.Add(1)
		go func(record pipeline.ChunkPlanRecord) {
			defer wg.Done()
			errs <- store.Save(record)
		}(records[i])
	}
	wg.Wait()
	close(errs)
	for err := range errs {
		if err != nil {
			t.Fatalf("concurrent Save() error = %v", err)
		}
	}
	got, decision, err := store.Load(testSourceDigest)
	if err != nil || decision.Status != pipeline.ChunkPlanHit {
		t.Fatalf("Load() decision=%#v error=%v", decision, err)
	}
	var annotation struct {
		Value int `json:"value"`
	}
	if err := json.Unmarshal(got.Plan.Annotations["test/value"], &annotation); err != nil || annotation.Value < 1 || annotation.Value > writers {
		t.Fatalf("final annotation=%#v error=%v", annotation, err)
	}
}

func TestFilesystemStoreReadersObserveOnlyCompleteRecordsDuringWrites(t *testing.T) {
	store := newStore(t, t.TempDir())
	if err := store.Save(testRecord(t, 1)); err != nil {
		t.Fatal(err)
	}
	const writers = 12
	const readers = 12
	errs := make(chan error, writers+readers)
	start := make(chan struct{})
	var writersDone sync.WaitGroup
	for i := 0; i < writers; i++ {
		writersDone.Add(1)
		go func(value int) {
			defer writersDone.Done()
			<-start
			errs <- store.Save(testRecord(t, value+2))
		}(i)
	}
	for i := 0; i < readers; i++ {
		go func() {
			<-start
			for attempt := 0; attempt < 50; attempt++ {
				record, decision, err := store.Load(testSourceDigest)
				if err != nil || decision.Status != pipeline.ChunkPlanHit {
					errs <- fmt.Errorf("Load() decision=%#v error=%v", decision, err)
					return
				}
				if err := validateRecord(record, testSourceDigest); err != nil {
					errs <- fmt.Errorf("reader observed invalid record: %w", err)
					return
				}
			}
			errs <- nil
		}()
	}
	close(start)
	writersDone.Wait()
	for i := 0; i < readers+writers; i++ {
		if err := <-errs; err != nil {
			t.Fatal(err)
		}
	}
}

func TestFilesystemStoreInterruptedWritesPreservePreviousRecord(t *testing.T) {
	store := newStore(t, t.TempDir()).(*filesystemStore)
	first := testRecord(t, 1)
	if err := store.Save(first); err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct {
		name  string
		hooks atomicWriteHooks
	}{
		{name: "before temporary file", hooks: atomicWriteHooks{BeforeCreateTemp: func() error { return errors.New("interrupted before temporary file") }}},
		{name: "before rename", hooks: atomicWriteHooks{BeforeRename: func() error { return errors.New("interrupted before rename") }}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			store.write = func(root *os.Root, target string, data []byte) error {
				return writeAtomicWithHooks(root, target, data, tc.hooks)
			}
			if err := store.Save(testRecord(t, 2)); err == nil {
				t.Fatal("Save() error = nil")
			}
			store.write = writeAtomic
			got, decision, err := store.Load(testSourceDigest)
			if err != nil || decision.Status != pipeline.ChunkPlanHit || !reflect.DeepEqual(got, first) {
				t.Fatalf("record after interruption=%#v decision=%#v error=%v", got, decision, err)
			}
		})
	}
}

func newStore(t *testing.T, root string) pipeline.ChunkPlanStore {
	t.Helper()
	store, err := NewFilesystemStore(root)
	if err != nil {
		t.Fatalf("NewFilesystemStore() error = %v", err)
	}
	return store
}

func testRecord(t *testing.T, value int) pipeline.ChunkPlanRecord {
	t.Helper()
	annotation, err := json.Marshal(map[string]int{"value": value})
	if err != nil {
		t.Fatal(err)
	}
	plan := source.ChunkPlan{
		SourceDigest: testSourceDigest,
		Ranges:       []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2, Annotations: source.ChunkAnnotations{"test/range": json.RawMessage(`{"range":true}`)}}},
		Annotations:  source.ChunkAnnotations{"test/value": annotation},
	}
	planDigest, err := source.DigestChunkPlan(plan)
	if err != nil {
		t.Fatal(err)
	}
	return pipeline.ChunkPlanRecord{
		SchemaVersion: SchemaVersion,
		SourceDigest:  testSourceDigest,
		PlanDigest:    planDigest,
		Plan:          plan,
		Producer: pipeline.ChunkPlanProducer{
			InputModule: "input/test", ChunkModule: "chunk/test", LLMProfile: "profile/test",
			References: []artifacts.ReferenceProvenance{{Stage: "chunk", SlotName: "guide", OriginType: "file", OriginURI: "file:///guide.txt", Digest: "sha256:reference"}},
			Metadata:   map[string]any{"prompt_id": "test/prompt", "enabled": true},
		},
		Diagnostics: []contracts.ProducerDiagnostic{{
			Disposition:     contracts.DiagnosticDispositionWarning,
			Category:        contracts.DiagnosticCategoryConfiguration,
			ReasonCode:      "empty_reference",
			OccurrenceCount: 1,
			Samples:         []contracts.DiagnosticSample{{Scope: "reference", Message: "Reference was empty."}},
		}},
		CreatedAt: time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC),
	}
}

func replaceJSON(old, replacement string) func([]byte) []byte {
	return func(data []byte) []byte { return bytes.Replace(data, []byte(old), []byte(replacement), 1) }
}

func assertNoTemps(t *testing.T, dir string) {
	t.Helper()
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatal(err)
	}
	for _, entry := range entries {
		if strings.Contains(entry.Name(), ".tmp-") {
			t.Fatalf("temporary file remains: %s", entry.Name())
		}
	}
}

func createSymlink(t *testing.T, target, link string) {
	t.Helper()
	if err := os.Symlink(target, link); err != nil {
		t.Skipf("create symlink: %v", err)
	}
}

func writeFile(t *testing.T, path string, data []byte, mode os.FileMode) {
	t.Helper()
	if err := os.WriteFile(path, data, mode); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(path, mode); err != nil {
		t.Fatal(err)
	}
}

func readFile(t *testing.T, path string) []byte {
	t.Helper()
	data, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func fileMode(t *testing.T, path string) os.FileMode {
	t.Helper()
	info, err := os.Stat(path)
	if err != nil {
		t.Fatal(err)
	}
	return info.Mode().Perm()
}

func outsidePlanPath(name, outside string) string {
	if name == "digest directory" {
		return filepath.Join(outside, "plan.json")
	}
	return outside
}