From 781341a0ba9d0f22c602d2175ddfece5f0f8c0db Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 2 Nov 2026 22:35:53 +0000 Subject: [PATCH] Preserve exact evidence bytes in private publication records --- docs/integrations/publication-records.md | 14 ++++- docs/roadmap/remediation.md | 7 +++ internal/publish/record.go | 67 ++++++++++++++++++-- internal/publish/record_test.go | 78 +++++++++++++++++++++++- 4 files changed, 157 insertions(+), 9 deletions(-) diff --git a/docs/integrations/publication-records.md b/docs/integrations/publication-records.md index 8c1d84e..3a37751 100644 --- a/docs/integrations/publication-records.md +++ b/docs/integrations/publication-records.md @@ -34,8 +34,18 @@ stopped before they were captured; loading preserves that partial evidence and does not fill fields from the current checkout. Additional fields in a supported version are ignored. A missing or unsupported version, malformed or multiple JSON documents, or fields with invalid structural types are reported -as record-format errors. The codec supports version 1; no migration from an -unsupported version is performed. +as record-format errors. The codec supports versions 1 and 2; no migration from +an unsupported version is performed. + +Ordinary UTF-8 evidence uses version 1. Evidence containing other pathname or +endpoint bytes uses version 2 with the same `evidence` object and an additional +`evidence_bytes` object. Each entry in `evidence_bytes` uses an evidence field +name and a standard base64 string containing its exact bytes. That entry +supplies the field's value when loading; the writer omits its lossy string form +from `evidence`. Unknown field names are ignored. This representation applies +to every evidence field and preserves the same identity as typed Go evidence. +Older CLIs that support only version 1 report an unsupported version for these +byte-containing records; use a CLI supporting version 2 to load them. The `endpoint` value is stored exactly as captured and may contain credentials. Private records are sensitive. Do not copy them into public logs or source diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 1edc34f..35e965a 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -384,6 +384,13 @@ documentation. ## R10: Preserve endpoint bytes across the private-record handoff — P2 +**Completed:** Private records retain ordinary version-1 JSON and select a +version-2 base64 byte representation for non-UTF-8 fields. Both versions load; +unknown fields remain ignorable and malformed known byte fields report format +errors. Round-trip tests passed for all evidence fields, and real Git +observations distinguish the original byte path from a replacement-character +decoy consistently before and after loading. The record-format guide is updated. + **Relevant code:** `internal/publish/record.go:77` and its evidence decoding; the string fields in `internal/publish/reconcile.go`'s `Evidence`; the existing Git endpoint resolution/query path in `internal/git/remote.go` and `tag.go`. diff --git a/internal/publish/record.go b/internal/publish/record.go index 0980546..f1a9ec4 100644 --- a/internal/publish/record.go +++ b/internal/publish/record.go @@ -12,15 +12,18 @@ import ( "path/filepath" "strings" "time" + "unicode/utf8" "go.maximumdirect.net/releasekit/internal/git" ) const recordFormatVersion = 1 +const recordByteFormatVersion = 2 type recordEnvelope struct { FormatVersion int `json:"format_version"` Evidence json.RawMessage `json:"evidence"` + EvidenceBytes json.RawMessage `json:"evidence_bytes,omitempty"` } // LoadEvidence reads one complete supported private publication record. @@ -38,7 +41,7 @@ func LoadEvidence(path string) (Evidence, error) { if err := decoder.Decode(&trailing); err != io.EOF { return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("record %q contains trailing JSON", path)} } - if envelope.FormatVersion != recordFormatVersion { + if envelope.FormatVersion != recordFormatVersion && envelope.FormatVersion != recordByteFormatVersion { return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("record %q has unsupported format version %d", path, envelope.FormatVersion)} } if len(envelope.Evidence) == 0 || string(envelope.Evidence) == "null" { @@ -48,9 +51,65 @@ func LoadEvidence(path string) (Evidence, error) { if err := json.Unmarshal(envelope.Evidence, &evidence); err != nil { return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record evidence %q: %w", path, err)} } + if envelope.FormatVersion == recordByteFormatVersion && len(envelope.EvidenceBytes) > 0 { + var fields map[string]json.RawMessage + if err := json.Unmarshal(envelope.EvidenceBytes, &fields); err != nil { + return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record evidence bytes %q: %w", path, err)} + } + for _, field := range evidenceFields(&evidence) { + if raw, ok := fields[field.name]; ok { + var bytes []byte + if err := json.Unmarshal(raw, &bytes); err != nil { + return Evidence{}, &RecoveryError{Kind: "record_format", RecordPath: path, cause: fmt.Errorf("decode record field %s bytes %q: %w", field.name, path, err)} + } + *field.value = string(bytes) + } + } + } return evidence, nil } +type evidenceField struct { + name string + value *string +} + +func evidenceFields(e *Evidence) [10]evidenceField { + return [10]evidenceField{ + {"repository_dir", &e.RepositoryDir}, {"module_dir", &e.ModuleDir}, + {"module_path", &e.ModulePath}, {"version", &e.Version}, {"tag", &e.Tag}, + {"candidate_commit", &e.CandidateCommit}, {"tag_object", &e.TagObject}, + {"endpoint", &e.Endpoint}, {"remote", &e.Remote}, {"recorded_at", &e.RecordedAt}, + } +} + +func encodeRecord(evidence Evidence) ([]byte, error) { + envelope := recordEnvelope{FormatVersion: recordFormatVersion} + var bytes map[string][]byte + for _, field := range evidenceFields(&evidence) { + if !utf8.ValidString(*field.value) { + if bytes == nil { + bytes = make(map[string][]byte) + } + bytes[field.name] = []byte(*field.value) + *field.value = "" + } + } + var err error + if bytes != nil { + envelope.FormatVersion = recordByteFormatVersion + envelope.EvidenceBytes, err = json.Marshal(bytes) + if err != nil { + return nil, err + } + } + envelope.Evidence, err = json.Marshal(evidence) + if err != nil { + return nil, err + } + return json.Marshal(envelope) +} + // SaveEvidence writes one private attempt record and returns its absolute path. func SaveEvidence(ctx context.Context, repositoryDir, override string, evidence Evidence) (string, error) { recordDir, root, err := resolveRecordDir(ctx, repositoryDir, override) @@ -71,11 +130,7 @@ func saveEvidenceNamed(ctx context.Context, repositoryDir, root, override, recor if err := os.MkdirAll(recordDir, 0o700); err != nil { return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("create record directory %q: %w", recordDir, err)} } - evidenceJSON, err := json.Marshal(evidence) - if err != nil { - return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("encode publication evidence: %w", err)} - } - data, err := json.Marshal(recordEnvelope{FormatVersion: recordFormatVersion, Evidence: evidenceJSON}) + data, err := encodeRecord(evidence) if err != nil { return "", &RecoveryError{Kind: "record_write", cause: fmt.Errorf("encode publication evidence: %w", err)} } diff --git a/internal/publish/record_test.go b/internal/publish/record_test.go index 5d2aadc..aef1037 100644 --- a/internal/publish/record_test.go +++ b/internal/publish/record_test.go @@ -8,11 +8,87 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strings" "testing" ) +func TestRecordsRoundTripNonUTF8Evidence(t *testing.T) { + repo := recordRepo(t) + raw := "path-\xff" + evidence := Evidence{RepositoryDir: raw, ModuleDir: raw, ModulePath: raw, Version: raw, Tag: raw, CandidateCommit: raw, TagObject: raw, Endpoint: raw, Remote: raw, RecordedAt: raw} + path, err := SaveEvidence(t.Context(), repo, t.TempDir(), evidence) + if err != nil { + t.Fatal(err) + } + loaded, err := LoadEvidence(path) + if err != nil || loaded != evidence { + t.Fatalf("lossy evidence: %#v, %v", loaded, err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var envelope recordEnvelope + if err := json.Unmarshal(data, &envelope); err != nil || envelope.FormatVersion != recordByteFormatVersion { + t.Fatalf("byte record envelope: %#v, %v", envelope, err) + } + // Unknown fields remain ignorable; known byte fields retain real codec errors. + for _, test := range []struct { + field string + wantError bool + }{{"endpoint", true}, {"future_field", false}} { + envelope.EvidenceBytes = json.RawMessage(`{"` + test.field + `":"not base64!"}`) + data, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatal(err) + } + _, err = LoadEvidence(path) + if (err != nil) != test.wantError { + t.Fatalf("byte field %s error=%v", test.field, err) + } + } +} + +func TestRecordLoadedObservationKeepsNonUTF8EndpointIdentity(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("raw pathname bytes are a POSIX behavior") + } + repo := recordRepo(t) + base := t.TempDir() + original := filepath.Join(base, "remote-\xff.git") + decoy := filepath.Join(base, "remote-\uFFFD.git") + for _, remote := range []string{original, decoy} { + recordGit(t, repo, "init", "--bare", remote) + } + recordGit(t, repo, "tag", "-a", "v1.2.3", "-m", "Release") + recordGit(t, repo, "push", decoy, "refs/tags/v1.2.3") + evidence := Evidence{RepositoryDir: repo, ModuleDir: repo, Endpoint: original, Tag: "v1.2.3", TagObject: strings.TrimSpace(recordGit(t, repo, "rev-parse", "v1.2.3")), CandidateCommit: strings.TrimSpace(recordGit(t, repo, "rev-parse", "HEAD"))} + path, err := SaveEvidence(t.Context(), repo, "", evidence) + if err != nil { + t.Fatal(err) + } + loaded, err := LoadEvidence(path) + if err != nil || loaded.Endpoint != original { + t.Fatalf("endpoint changed: %q, %v", loaded.Endpoint, err) + } + for _, selected := range []Evidence{evidence, loaded} { + observed, err := Reconcile(t.Context(), selected) + if err != nil || observed.State != NotPublished { + t.Fatalf("original endpoint observation = %#v, %v", observed, err) + } + } + loaded.Endpoint = decoy + observed, err := Reconcile(t.Context(), loaded) + if err != nil || observed.State != Published { + t.Fatalf("decoy does not distinguish identity: %#v, %v", observed, err) + } +} + func recordGit(t *testing.T, dir string, args ...string) string { t.Helper() cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) @@ -47,7 +123,7 @@ func TestLoadEvidenceAcceptsPartialAndRejectsMalformedRecords(t *testing.T) { if err != nil || got != (Evidence{}) { t.Fatalf("partial record = %#v, %v", got, err) } - for name, data := range map[string]string{"bad-json": "{", "unsupported": `{"format_version":2,"evidence":{}}`, "type": `{"format_version":1,"evidence":{"tag":false}}`, "trailing": `{"format_version":1,"evidence":{}} {}`} { + for name, data := range map[string]string{"bad-json": "{", "unsupported": `{"format_version":3,"evidence":{}}`, "type": `{"format_version":1,"evidence":{"tag":false}}`, "trailing": `{"format_version":1,"evidence":{}} {}`} { t.Run(name, func(t *testing.T) { path := filepath.Join(dir, name) if err := os.WriteFile(path, []byte(data), 0600); err != nil {