Preserve exact evidence bytes in private publication records

docs/integrations/publication-records.md

@@ -34,8 +34,18 @@ stopped before they were captured; loading preserves that partial evidence and
 does not fill fields from the current checkout. Additional fields in a
 supported version are ignored. A missing or unsupported version, malformed or
 multiple JSON documents, or fields with invalid structural types are reported
-as record-format errors. The codec supports version 1; no migration from an
+as record-format errors. The codec supports versions 1 and 2; no migration from
-unsupported version is performed.
+an unsupported version is performed.
+
+Ordinary UTF-8 evidence uses version 1. Evidence containing other pathname or
+endpoint bytes uses version 2 with the same `evidence` object and an additional
+`evidence_bytes` object. Each entry in `evidence_bytes` uses an evidence field
+name and a standard base64 string containing its exact bytes. That entry
+supplies the field's value when loading; the writer omits its lossy string form
+from `evidence`. Unknown field names are ignored. This representation applies
+to every evidence field and preserves the same identity as typed Go evidence.
+Older CLIs that support only version 1 report an unsupported version for these
+byte-containing records; use a CLI supporting version 2 to load them.
 
 The `endpoint` value is stored exactly as captured and may contain credentials.
 Private records are sensitive. Do not copy them into public logs or source

docs/roadmap/remediation.md

@@ -384,6 +384,13 @@ documentation.
 
 ## R10: Preserve endpoint bytes across the private-record handoff — P2
 
+**Completed:** Private records retain ordinary version-1 JSON and select a
+version-2 base64 byte representation for non-UTF-8 fields. Both versions load;
+unknown fields remain ignorable and malformed known byte fields report format
+errors. Round-trip tests passed for all evidence fields, and real Git
+observations distinguish the original byte path from a replacement-character
+decoy consistently before and after loading. The record-format guide is updated.
+
 **Relevant code:** `internal/publish/record.go:77` and its evidence decoding;
 the string fields in `internal/publish/reconcile.go`'s `Evidence`; the existing
 Git endpoint resolution/query path in `internal/git/remote.go` and `tag.go`.

internal/publish/record.go

@@ -12,15 +12,18 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"go.maximumdirect.net/releasekit/internal/git"
 )
 
 const recordFormatVersion = 1
+const recordByteFormatVersion = 2
 
 type recordEnvelope struct {
 	FormatVersion int             `json:"format_version"`
 	Evidence      json.RawMessage `json:"evidence"`
+	EvidenceBytes json.RawMessage `json:"evidence_bytes,omitempty"`
 }
 
 // LoadEvidence reads one complete supported private publication record.
@@ -38,7 +41,7 @@ func LoadEvidence(path string) (Evidence, error) {
 	if err := decoder.Decode(&trailing); err != io.EOF {
 		return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("record %q contains trailing JSON", path)}
 	}
-	if envelope.FormatVersion != recordFormatVersion {
+	if envelope.FormatVersion != recordFormatVersion && envelope.FormatVersion != recordByteFormatVersion {
 		return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("record %q has unsupported format version %d", path, envelope.FormatVersion)}
 	}
 	if len(envelope.Evidence) == 0 || string(envelope.Evidence) == "null" {
@@ -48,9 +51,65 @@ func LoadEvidence(path string) (Evidence, error) {
 	if err := json.Unmarshal(envelope.Evidence, &evidence); err != nil {
 		return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record evidence %q: %w", path, err)}
 	}
+	if envelope.FormatVersion == recordByteFormatVersion && len(envelope.EvidenceBytes) > 0 {
+		var fields map[string]json.RawMessage
+		if err := json.Unmarshal(envelope.EvidenceBytes, &fields); err != nil {
+			return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record evidence bytes %q: %w", path, err)}
+		}
+		for _, field := range evidenceFields(&evidence) {
+			if raw, ok := fields[field.name]; ok {
+				var bytes []byte
+				if err := json.Unmarshal(raw, &bytes); err != nil {
+					return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record field %s bytes %q: %w", field.name, path, err)}
+				}
+				*field.value = string(bytes)
+			}
+		}
+	}
 	return evidence, nil
 }
 
+type evidenceField struct {
+	name  string
+	value *string
+}
+
+func evidenceFields(e *Evidence) [10]evidenceField {
+	return [10]evidenceField{
+		{"repository_dir", &e.RepositoryDir}, {"module_dir", &e.ModuleDir},
+		{"module_path", &e.ModulePath}, {"version", &e.Version}, {"tag", &e.Tag},
+		{"candidate_commit", &e.CandidateCommit}, {"tag_object", &e.TagObject},
+		{"endpoint", &e.Endpoint}, {"remote", &e.Remote}, {"recorded_at", &e.RecordedAt},
+	}
+}
+
+func encodeRecord(evidence Evidence) ([]byte, error) {
+	envelope := recordEnvelope{FormatVersion: recordFormatVersion}
+	var bytes map[string][]byte
+	for _, field := range evidenceFields(&evidence) {
+		if !utf8.ValidString(*field.value) {
+			if bytes == nil {
+				bytes = make(map[string][]byte)
+			}
+			bytes[field.name] = []byte(*field.value)
+			*field.value = ""
+		}
+	}
+	var err error
+	if bytes != nil {
+		envelope.FormatVersion = recordByteFormatVersion
+		envelope.EvidenceBytes, err = json.Marshal(bytes)
+		if err != nil {
+			return nil, err
+		}
+	}
+	envelope.Evidence, err = json.Marshal(evidence)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(envelope)
+}
+
 // SaveEvidence writes one private attempt record and returns its absolute path.
 func SaveEvidence(ctx context.Context, repositoryDir, override string, evidence Evidence) (string, error) {
 	recordDir, root, err := resolveRecordDir(ctx, repositoryDir, override)
@@ -71,11 +130,7 @@ func saveEvidenceNamed(ctx context.Context, repositoryDir, root, override, recor
 	if err := os.MkdirAll(recordDir, 0o700); err != nil {
 		return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("create record directory %q: %w", recordDir, err)}
 	}
-	evidenceJSON, err := json.Marshal(evidence)
+	data, err := encodeRecord(evidence)
-	if err != nil {
-		return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("encode publication evidence: %w", err)}
-	}
-	data, err := json.Marshal(recordEnvelope{FormatVersion: recordFormatVersion, Evidence: evidenceJSON})
 	if err != nil {
 		return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("encode publication evidence: %w", err)}
 	}

internal/publish/record_test.go

@@ -8,11 +8,87 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"testing"
 )
 
+func TestRecordsRoundTripNonUTF8Evidence(t *testing.T) {
+	repo := recordRepo(t)
+	raw := "path-\xff"
+	evidence := Evidence{RepositoryDir: raw, ModuleDir: raw, ModulePath: raw, Version: raw, Tag: raw, CandidateCommit: raw, TagObject: raw, Endpoint: raw, Remote: raw, RecordedAt: raw}
+	path, err := SaveEvidence(t.Context(), repo, t.TempDir(), evidence)
+	if err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := LoadEvidence(path)
+	if err != nil || loaded != evidence {
+		t.Fatalf("lossy evidence: %#v, %v", loaded, err)
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var envelope recordEnvelope
+	if err := json.Unmarshal(data, &envelope); err != nil || envelope.FormatVersion != recordByteFormatVersion {
+		t.Fatalf("byte record envelope: %#v, %v", envelope, err)
+	}
+	// Unknown fields remain ignorable; known byte fields retain real codec errors.
+	for _, test := range []struct {
+		field     string
+		wantError bool
+	}{{"endpoint", true}, {"future_field", false}} {
+		envelope.EvidenceBytes = json.RawMessage(`{"` + test.field + `":"not base64!"}`)
+		data, err := json.Marshal(envelope)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(path, data, 0600); err != nil {
+			t.Fatal(err)
+		}
+		_, err = LoadEvidence(path)
+		if (err != nil) != test.wantError {
+			t.Fatalf("byte field %s error=%v", test.field, err)
+		}
+	}
+}
+
+func TestRecordLoadedObservationKeepsNonUTF8EndpointIdentity(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("raw pathname bytes are a POSIX behavior")
+	}
+	repo := recordRepo(t)
+	base := t.TempDir()
+	original := filepath.Join(base, "remote-\xff.git")
+	decoy := filepath.Join(base, "remote-\uFFFD.git")
+	for _, remote := range []string{original, decoy} {
+		recordGit(t, repo, "init", "--bare", remote)
+	}
+	recordGit(t, repo, "tag", "-a", "v1.2.3", "-m", "Release")
+	recordGit(t, repo, "push", decoy, "refs/tags/v1.2.3")
+	evidence := Evidence{RepositoryDir: repo, ModuleDir: repo, Endpoint: original, Tag: "v1.2.3", TagObject: strings.TrimSpace(recordGit(t, repo, "rev-parse", "v1.2.3")), CandidateCommit: strings.TrimSpace(recordGit(t, repo, "rev-parse", "HEAD"))}
+	path, err := SaveEvidence(t.Context(), repo, "", evidence)
+	if err != nil {
+		t.Fatal(err)
+	}
+	loaded, err := LoadEvidence(path)
+	if err != nil || loaded.Endpoint != original {
+		t.Fatalf("endpoint changed: %q, %v", loaded.Endpoint, err)
+	}
+	for _, selected := range []Evidence{evidence, loaded} {
+		observed, err := Reconcile(t.Context(), selected)
+		if err != nil || observed.State != NotPublished {
+			t.Fatalf("original endpoint observation = %#v, %v", observed, err)
+		}
+	}
+	loaded.Endpoint = decoy
+	observed, err := Reconcile(t.Context(), loaded)
+	if err != nil || observed.State != Published {
+		t.Fatalf("decoy does not distinguish identity: %#v, %v", observed, err)
+	}
+}
+
 func recordGit(t *testing.T, dir string, args ...string) string {
 	t.Helper()
 	cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
@@ -47,7 +123,7 @@ func TestLoadEvidenceAcceptsPartialAndRejectsMalformedRecords(t *testing.T) {
 	if err != nil || got != (Evidence{}) {
 		t.Fatalf("partial record = %#v, %v", got, err)
 	}
-	for name, data := range map[string]string{"bad-json": "{", "unsupported": `{"format_version":2,"evidence":{}}`, "type": `{"format_version":1,"evidence":{"tag":false}}`, "trailing": `{"format_version":1,"evidence":{}} {}`} {
+	for name, data := range map[string]string{"bad-json": "{", "unsupported": `{"format_version":3,"evidence":{}}`, "type": `{"format_version":1,"evidence":{"tag":false}}`, "trailing": `{"format_version":1,"evidence":{}} {}`} {
 		t.Run(name, func(t *testing.T) {
 			path := filepath.Join(dir, name)
 			if err := os.WriteFile(path, []byte(data), 0600); err != nil {