Record debug evidence in one run-scoped capture session

This commit is contained in:
2026-11-22 02:49:48 +00:00
parent 584b041e57
commit 8680060400
10 changed files with 700 additions and 676 deletions

View File

@@ -1,38 +1,167 @@
// Package debug records explicitly requested application evidence.
package debug
import (
"strings"
"errors"
"fmt"
"sync"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type FilesystemRecorder struct {
root string
// Recorder writes evidence incrementally. It retains only lifecycle information
// and the first recording error, not the pipeline's content.
type Recorder struct {
bundle *debugbundle.Bundle
mu sync.Mutex
recordingErr error
finalized bool
finalErr error
}
func NewFilesystemRecorder(root string) (pipeline.DebugRecorder, error) {
root = strings.TrimSpace(root)
if strings.TrimSpace(root) == "" {
return pipeline.NoopDebugRecorder(), nil
func Start(parent, runID string, startedAt time.Time) (*Recorder, error) {
bundle, err := debugbundle.Allocate(parent, runID, startedAt)
if bundle == nil {
return nil, err
}
return &FilesystemRecorder{root: root}, nil
recorder := &Recorder{bundle: bundle}
if err != nil {
recorder.RecordError(err)
return recorder, err
}
if err := recorder.WriteJSON("run.json", struct {
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}{runID, startedAt}); err != nil {
return recorder, err
}
if err := recorder.WriteBytes("overview.txt", []byte(fmt.Sprintf("Run: %s\nStarted: %s\nCapture: %s\nStatus: running\n", runID, startedAt.Format(time.RFC3339Nano), recorder.Path()))); err != nil {
return recorder, err
}
return recorder, nil
}
func (r *FilesystemRecorder) Enabled() bool {
return r != nil && strings.TrimSpace(r.root) != ""
func (r *Recorder) Enabled() bool { return r != nil && r.bundle != nil }
func (r *Recorder) Path() string {
if r == nil {
return ""
}
return r.bundle.Path()
}
func (r *FilesystemRecorder) WriteJSON(name string, payload any) error {
// RecordError retains callback failures for a boundary that can return errors.
func (r *Recorder) RecordError(err error) {
if !r.Enabled() || err == nil {
return
}
r.mu.Lock()
defer r.mu.Unlock()
r.remember(err)
}
func (r *Recorder) remember(err error) {
if r.recordingErr == nil {
r.recordingErr = err
}
}
func (r *Recorder) Err() error {
if !r.Enabled() {
return nil
}
return fileio.WriteJSON(r.root, name, payload, 0o700, 0o600)
r.mu.Lock()
defer r.mu.Unlock()
return r.recordingErr
}
func (r *FilesystemRecorder) WriteBytes(name string, data []byte) error {
func (r *Recorder) WriteJSON(name string, payload any) error {
if !r.Enabled() {
return nil
}
return fileio.WriteBytes(r.root, name, data, 0o700, 0o600)
r.mu.Lock()
defer r.mu.Unlock()
if r.finalized {
return fmt.Errorf("debug capture %q is finalized", r.Path())
}
if err := fileio.WriteJSON(r.Path(), name, payload, 0o700, 0o600); err != nil {
r.remember(fmt.Errorf("record debug %q in %q: %w", name, r.Path(), err))
}
return r.recordingErr
}
func (r *Recorder) WriteBytes(name string, content []byte) error {
if !r.Enabled() {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
if r.finalized {
return fmt.Errorf("debug capture %q is finalized", r.Path())
}
if err := fileio.WriteBytes(r.Path(), name, content, 0o700, 0o600); err != nil {
r.remember(fmt.Errorf("record debug %q in %q: %w", name, r.Path(), err))
}
return r.recordingErr
}
// Finalize writes one terminal outcome, retaining a primary operation failure
// before additional capture failures. Repeated calls neither write nor retarget
// latest. An additional primary error is preserved alongside the original
// finalization result, including when presentation fails after capture finishes.
func (r *Recorder) Finalize(outcome any, primaryErr error) error {
if !r.Enabled() {
return primaryErr
}
r.mu.Lock()
defer r.mu.Unlock()
if r.finalized {
if primaryErr == nil {
return r.finalErr
}
if errors.Is(primaryErr, r.finalErr) {
return primaryErr
}
return errors.Join(primaryErr, r.finalErr)
}
r.finalized = true
completedAt := time.Now().UTC()
terminal := struct {
CompletedAt time.Time `json:"completed_at"`
Outcome any `json:"outcome,omitempty"`
Error string `json:"error,omitempty"`
CaptureError string `json:"capture_error,omitempty"`
}{CompletedAt: completedAt, Outcome: outcome}
if primaryErr != nil {
terminal.Error = primaryErr.Error()
}
if r.recordingErr != nil {
terminal.CaptureError = r.recordingErr.Error()
}
outcomeErr := fileio.WriteJSON(r.Path(), "outcome.json", terminal, 0o700, 0o600)
if outcomeErr != nil {
outcomeErr = fmt.Errorf("finalize debug outcome in %q: %w", r.Path(), outcomeErr)
}
status := "completed"
if primaryErr != nil || r.recordingErr != nil || outcomeErr != nil {
status = "failed"
}
overview := fmt.Sprintf("Run: %s\nStarted: %s\nCompleted: %s\nCapture: %s\nStatus: %s\n", r.bundle.RunID(), r.bundle.CreatedAt().Format(time.RFC3339Nano), completedAt.Format(time.RFC3339Nano), r.Path(), status)
if primaryErr != nil {
overview += "Error: " + primaryErr.Error() + "\n"
}
if r.recordingErr != nil {
overview += "Capture error: " + r.recordingErr.Error() + "\n"
}
if outcomeErr != nil {
overview += "Capture error: " + outcomeErr.Error() + "\n"
}
overviewErr := fileio.WriteBytes(r.Path(), "overview.txt", []byte(overview), 0o700, 0o600)
if overviewErr != nil {
overviewErr = fmt.Errorf("finalize debug overview in %q: %w", r.Path(), overviewErr)
}
captureErr := r.recordingErr
if primaryErr != nil && errors.Is(primaryErr, captureErr) {
captureErr = nil
}
r.finalErr = errors.Join(primaryErr, captureErr, outcomeErr, overviewErr)
return r.finalErr
}

View File

@@ -1,79 +1,210 @@
package debug
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestFilesystemRecorderWritesWithinTraceRoot(t *testing.T) {
root := t.TempDir()
recorder, err := NewFilesystemRecorder(root)
func startTestCapture(t *testing.T, parent, id string) *Recorder {
t.Helper()
recorder, err := Start(parent, id, time.Unix(42, 0).UTC())
if err != nil {
t.Fatal(err)
}
if err := recorder.WriteBytes("attempt/data", []byte("payload")); err != nil {
return recorder
}
func TestRecorderPersistsIncrementallyAndFinalizesOnce(t *testing.T) {
parent := t.TempDir()
recorder := startTestCapture(t, parent, "first")
var _ pipeline.DebugRecorder = recorder
source := strings.Repeat("Source text and sk-ordinary-application-content. ", 400)
if err := recorder.WriteBytes("operations/extract/source.txt", []byte(source)); err != nil {
t.Fatal(err)
}
info, err := os.Stat(filepath.Join(root, "attempt", "data"))
if err != nil {
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "operations/extract/source.txt")); err != nil || string(got) != source {
t.Fatalf("incremental evidence=%q %v", got, err)
}
second := startTestCapture(t, parent, "second")
primary := errors.New("pipeline failed")
result := struct {
OutputDirectory string
Published bool
}{"/output/already-published", true}
if err := recorder.Finalize(result, primary); !errors.Is(err, primary) {
t.Fatalf("primary failure lost: %v", err)
}
before, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
if err != nil || !strings.Contains(string(before), result.OutputDirectory) || !strings.Contains(string(before), primary.Error()) {
t.Fatalf("terminal evidence=%s %v", before, err)
}
if err := recorder.Finalize("replacement", nil); !errors.Is(err, primary) {
t.Fatalf("repeat finalization=%v", err)
}
after, err := os.ReadFile(filepath.Join(recorder.Path(), "outcome.json"))
if err != nil || string(after) != string(before) {
t.Fatalf("terminal outcome rewritten: %q %v", after, err)
}
if err := second.Finalize(map[string]any{"success": true}, nil); err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("mode=%#o", info.Mode().Perm())
deliveryErr := errors.New("stdout delivery failed after publication")
if err := second.Finalize(nil, deliveryErr); !errors.Is(err, deliveryErr) {
t.Fatalf("post-capture delivery failure lost: %v", err)
}
if err := recorder.WriteBytes("../outside", nil); err == nil {
t.Fatal("accepted traversal")
target, err := os.Readlink(filepath.Join(parent, "latest"))
if err != nil || target != "second" {
t.Fatalf("completion retargeted latest: %q %v", target, err)
}
if err := recorder.WriteBytes("late.txt", nil); err == nil {
t.Fatal("write after finalization succeeded")
}
info, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json"))
if err != nil || info.Mode().Perm() != 0o600 {
t.Fatalf("capture file=%v %v", info, err)
}
}
func TestFilesystemRecorderPropagatesWriteFailures(t *testing.T) {
root := filepath.Join(t.TempDir(), "trace")
if err := os.WriteFile(root, []byte("not a directory"), 0o600); err != nil {
func TestRecorderRetainsCallbackErrorAndExposesItAtBoundary(t *testing.T) {
recorder := startTestCapture(t, t.TempDir(), "run")
callbackErr := errors.New("observer recording failed")
recorder.RecordError(callbackErr)
if !errors.Is(recorder.Err(), callbackErr) {
t.Fatal("callback failure not retained")
}
if err := recorder.WriteJSON("available.json", map[string]string{"available": "evidence"}); !errors.Is(err, callbackErr) {
t.Fatalf("boundary hid callback error: %v", err)
}
if _, err := os.Stat(filepath.Join(recorder.Path(), "available.json")); err != nil {
t.Fatal(err)
}
recorder, err := NewFilesystemRecorder(root)
if err != nil {
t.Fatal(err)
}
if err := recorder.WriteBytes("attempt/data", []byte("payload")); err == nil {
t.Fatal("WriteBytes concealed a trace-root write failure")
primary := errors.New("primary operation failed")
err := recorder.Finalize(map[string]string{"output_directory": "/published"}, primary)
if !errors.Is(err, primary) || !errors.Is(err, callbackErr) || !strings.HasPrefix(err.Error(), primary.Error()) {
t.Fatalf("failure causes/order lost: %v", err)
}
}
func TestSynchronizedFilesystemRecorderSupportsConcurrentWrites(t *testing.T) {
func TestRecorderWriteAndFinalizationFailuresPreserveEvidence(t *testing.T) {
for _, defect := range []string{"incremental write", "outcome write", "overview write", "outcome encoding"} {
t.Run(defect, func(t *testing.T) {
recorder := startTestCapture(t, t.TempDir(), "run")
if err := recorder.WriteBytes("available.txt", []byte("retained")); err != nil {
t.Fatal(err)
}
var result any = map[string]any{"output_directory": "/published", "artifact_count": 2}
switch defect {
case "incremental write":
if err := os.WriteFile(filepath.Join(recorder.Path(), "operations"), nil, 0o600); err != nil {
t.Fatal(err)
}
if err := recorder.WriteJSON("operations/result.json", result); err == nil {
t.Fatal("write failure hidden")
}
case "outcome write":
if err := os.Mkdir(filepath.Join(recorder.Path(), "outcome.json"), 0o700); err != nil {
t.Fatal(err)
}
case "overview write":
// Replace only the initial overview file with a directory to make the
// terminal write fail, without relying on effective UID permissions.
filename := filepath.Join(recorder.Path(), "overview.txt")
if err := os.Rename(filename, filepath.Join(recorder.Path(), "initial-overview.txt")); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filename, 0o700); err != nil {
t.Fatal(err)
}
case "outcome encoding":
result = make(chan int)
}
finalErr := recorder.Finalize(result, nil)
if finalErr == nil {
t.Fatal("capture failure reported success")
}
if got, err := os.ReadFile(filepath.Join(recorder.Path(), "available.txt")); err != nil || string(got) != "retained" {
t.Fatalf("evidence removed: %q %v", got, err)
}
primary := errors.New("new failure")
if err := recorder.Finalize(nil, primary); !errors.Is(err, primary) || !errors.Is(err, finalErr) {
t.Fatalf("additional primary error or original capture failure lost: %v", err)
}
})
}
}
func TestRecorderStartFailureProvidesAvailableDirectory(t *testing.T) {
root := t.TempDir()
recorder, err := NewFilesystemRecorder(root)
if err != nil {
if err := os.Mkdir(filepath.Join(root, "latest"), 0o700); err != nil {
t.Fatal(err)
}
recorder = pipeline.SynchronizedDebugRecorder(recorder)
recorder, err := Start(root, "run", time.Now())
if err == nil || recorder == nil || recorder.Path() != filepath.Join(root, "run") {
t.Fatalf("start=%#v %v", recorder, err)
}
if err := recorder.Finalize(nil, err); err == nil {
t.Fatal("start failure concealed")
}
if _, err := os.Stat(filepath.Join(recorder.Path(), "outcome.json")); err != nil {
t.Fatal(err)
}
}
const writes = 16
func TestRecorderConcurrentWritesAndBoundaryChecks(t *testing.T) {
recorder := startTestCapture(t, t.TempDir(), "run")
const count = 16
var wg sync.WaitGroup
errs := make(chan error, writes)
for i := 0; i < writes; i++ {
i := i
failures := make(chan error, count)
for i := range count {
wg.Add(1)
go func() {
go func(i int) {
defer wg.Done()
errs <- recorder.WriteBytes(fmt.Sprintf("attempt/%02d/data", i), []byte("payload"))
}()
failures <- recorder.WriteJSON(fmt.Sprintf("operations/%02d/result.json", i), map[string]int{"value": i})
_ = recorder.Err()
}(i)
}
wg.Wait()
close(errs)
for err := range errs {
close(failures)
for err := range failures {
if err != nil {
t.Fatal(err)
}
}
for i := 0; i < writes; i++ {
if _, err := os.Stat(filepath.Join(root, "attempt", fmt.Sprintf("%02d", i), "data")); err != nil {
t.Fatalf("trace artifact %d: %v", i, err)
if err := recorder.Finalize(map[string]int{"operations": count}, nil); err != nil {
t.Fatal(err)
}
for i := range count {
content, err := os.ReadFile(filepath.Join(recorder.Path(), fmt.Sprintf("operations/%02d/result.json", i)))
var decoded map[string]int
if err != nil || json.Unmarshal(content, &decoded) != nil || decoded["value"] != i {
t.Fatalf("operation %d: %s %v", i, content, err)
}
}
}
func TestDisabledRecorderIsANoopAndPreservesPrimaryFailure(t *testing.T) {
var recorder *Recorder
primary := errors.New("operation failed")
if recorder.Enabled() || recorder.Path() != "" || recorder.Err() != nil {
t.Fatal("nil recorder enabled")
}
if err := recorder.WriteJSON("../ignored", make(chan int)); err != nil {
t.Fatal(err)
}
if err := recorder.WriteBytes("../ignored", nil); err != nil {
t.Fatal(err)
}
recorder.RecordError(errors.New("ignored"))
if err := recorder.Finalize(nil, primary); err != primary {
t.Fatalf("primary=%v", err)
}
}