|
|
|
|
@@ -0,0 +1,200 @@
|
|
|
|
|
package app
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"errors"
|
|
|
|
|
"os"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/storage"
|
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/adapters/whisperx"
|
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
|
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/config"
|
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
|
|
|
|
|
"go.maximumdirect.net/commandkit"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func executeProgressCommand(t *testing.T, app *commandkit.App, args []string) (runReport, string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var out, diagnostic bytes.Buffer
|
|
|
|
|
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
|
|
|
|
|
if outcome.Status != 0 {
|
|
|
|
|
t.Fatalf("outcome=%#v diagnostics=%s", outcome, diagnostic.String())
|
|
|
|
|
}
|
|
|
|
|
var report runReport
|
|
|
|
|
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
|
|
|
|
t.Fatalf("expected one final JSON report: %v; %s", err, out.String())
|
|
|
|
|
}
|
|
|
|
|
if !report.Complete {
|
|
|
|
|
t.Fatalf("incomplete report=%#v", report)
|
|
|
|
|
}
|
|
|
|
|
return report, diagnostic.String()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestNativeProgressTracksDownloadsConcurrentSpeakersAndReuse(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
pipeline, campaign, session := writeValidConfigFiles(t, root)
|
|
|
|
|
data, err := os.ReadFile(pipeline)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
mustWriteTestFile(t, pipeline, strings.Replace(string(data), "concurrency: 1", "concurrency: 2", 1))
|
|
|
|
|
mustWriteTestFile(t, session, "session_id: 2026-05-03\ninputs:\n audio_s3: {prefix: audio/}\n")
|
|
|
|
|
store := &storage.FakeBackend{}
|
|
|
|
|
loaded, err := config.LoadPipelineWithOptions(pipeline, config.PipelineLoadOptions{})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
prefix := artifacts.S3SessionPrefix(loaded.Storage.S3.RootPrefix, "sample-campaign", "2026-05-03")
|
|
|
|
|
for _, speaker := range []string{"a", "b", "c", "d"} {
|
|
|
|
|
store.SeedObject(storage.FakeObject{Key: prefix + "audio/" + speaker + ".flac", Data: []byte("synthetic-audio-" + speaker), ETag: "synthetic-generation"})
|
|
|
|
|
}
|
|
|
|
|
transcriber := &whisperx.FakeClient{TranscribeFn: func(_ context.Context, req whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
|
|
|
|
|
return writeTranscribeRecoveryResult(req)
|
|
|
|
|
}}
|
|
|
|
|
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
|
|
|
opts.Env = &Env{WhisperX: transcriber, ObjectStore: store}
|
|
|
|
|
return executePlan(ctx, cfg, plan, opts)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
common := []string{"--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json", "--progress"}
|
|
|
|
|
_, diagnostic := executeProgressCommand(t, app, append([]string{"run", "2026-05-03", "--from", "prepare", "--through", "transcribe"}, common...))
|
|
|
|
|
if strings.Count(diagnostic, "downloading audio") != 4 || strings.Count(diagnostic, "audio downloaded") != 4 || strings.Count(diagnostic, "speaker transcription completed") != 4 {
|
|
|
|
|
t.Fatalf("missing actual work progress: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
if strings.Contains(diagnostic, "selected audio source") || strings.Contains(diagnostic, "speaker transcription output") {
|
|
|
|
|
t.Fatalf("progress-only enabled contextual information: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
if len(store.Downloads) != 4 || len(transcriber.RequestsSnapshot()) != 4 {
|
|
|
|
|
t.Fatal("progress did not correspond to actual external work")
|
|
|
|
|
}
|
|
|
|
|
manifestPath := artifacts.SessionManifestPathForCampaign(root, "sample-campaign", "2026-05-03")
|
|
|
|
|
saved, err := (&manifest.LocalStore{}).Load(context.Background(), manifestPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
paths := artifacts.NewLocalStore(root).SessionPathsFor("sample-campaign", "2026-05-03")
|
|
|
|
|
mustWriteTestFile(t, filepath.Join(paths.Root, filepath.FromSlash(saved.Stages["transcribe"].TranscribeSpeakers["b"].Output.LocalPath)), "corrupt")
|
|
|
|
|
transcriber = &whisperx.FakeClient{TranscribeFn: func(_ context.Context, req whisperx.TranscribeRequest) (whisperx.TranscribeResult, error) {
|
|
|
|
|
return writeTranscribeRecoveryResult(req)
|
|
|
|
|
}}
|
|
|
|
|
args := append([]string{"run", "2026-05-03", "--stage", "transcribe", "--verbose"}, common...)
|
|
|
|
|
_, diagnostic = executeProgressCommand(t, app, args)
|
|
|
|
|
if strings.Count(diagnostic, "reusing speaker transcription") != 3 || strings.Count(diagnostic, "speaker transcription completed") != 1 || !strings.Contains(diagnostic, "speaker transcription reuse decision") {
|
|
|
|
|
t.Fatalf("missing partial reuse/completion context: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
if got := requestedTranscribeSpeakers(transcriber); len(got) != 1 || got[0] != "b" {
|
|
|
|
|
t.Fatalf("unexpected repeated work=%v", got)
|
|
|
|
|
}
|
|
|
|
|
_, diagnostic = executeProgressCommand(t, app, append([]string{"run", "2026-05-03", "--stage", "prepare", "--force"}, common...))
|
|
|
|
|
if strings.Count(diagnostic, "reusing cached audio") != 4 || strings.Contains(diagnostic, "downloading audio") || len(store.Downloads) != 4 {
|
|
|
|
|
t.Fatalf("cache reuse claimed new downloads: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
store.SeedObject(storage.FakeObject{Key: prefix + "audio/a.flac", Data: []byte("replacement-audio"), ETag: "new-generation"})
|
|
|
|
|
cause := errors.New("audio download failed")
|
|
|
|
|
store.DownloadErr = cause
|
|
|
|
|
var out, diagnostics bytes.Buffer
|
|
|
|
|
outcome := app.Execute(context.Background(), append([]string{"run", "2026-05-03", "--stage", "prepare", "--force"}, common...), commandkit.Streams{Out: &out, Err: &diagnostics})
|
|
|
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Count(diagnostics.String(), "downloading audio") != 1 || strings.Contains(diagnostics.String(), "audio downloaded") {
|
|
|
|
|
t.Fatalf("failed download progress: outcome=%#v diagnostics=%s", outcome, diagnostics.String())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestNativeProgressTracksArtifactCheckpointsAndReuse(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
pipeline, campaign, session := writeArtifactProgressConfig(t, root)
|
|
|
|
|
path := artifacts.SessionManifestPathForCampaign(root, "sample-campaign", "2026-05-03")
|
|
|
|
|
runner := &analyzeCheckpointRunner{}
|
|
|
|
|
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
|
|
|
opts.Env = &Env{Scriptorium: runner}
|
|
|
|
|
return executePlan(ctx, cfg, plan, opts)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
args := []string{"run", "2026-05-03", "--stage", "analyze", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--format", "json", "--progress"}
|
|
|
|
|
_, diagnostic := executeProgressCommand(t, app, args)
|
|
|
|
|
if strings.Count(diagnostic, "artifact completed") != 2 || strings.Contains(diagnostic, "artifact output") {
|
|
|
|
|
t.Fatalf("incorrect progress-only artifact messages: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
saved, err := (&manifest.LocalStore{}).Load(context.Background(), path)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
paths := artifacts.NewLocalStore(root).SessionPathsFor("sample-campaign", "2026-05-03")
|
|
|
|
|
mustWriteTestFile(t, filepath.Join(paths.Root, filepath.FromSlash(saved.Stages["analyze"].AnalyzeArtifacts["beta"].Output.LocalPath)), "changed")
|
|
|
|
|
runner = &analyzeCheckpointRunner{}
|
|
|
|
|
_, diagnostic = executeProgressCommand(t, app, append(args, "--verbose"))
|
|
|
|
|
if strings.Count(diagnostic, "reusing artifact") != 1 || strings.Count(diagnostic, "artifact completed") != 1 || !strings.Contains(diagnostic, "artifact reuse decision") || !strings.Contains(diagnostic, "artifact output") {
|
|
|
|
|
t.Fatalf("incorrect artifact reuse/completion messages: %s", diagnostic)
|
|
|
|
|
}
|
|
|
|
|
if len(runner.Calls) != 1 || runner.Calls[0] != "beta" {
|
|
|
|
|
t.Fatalf("repeated valid artifact work=%v", runner.Calls)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func writeArtifactProgressConfig(t *testing.T, root string) (string, string, string) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
pipeline, campaign, session := writeValidConfigFiles(t, root)
|
|
|
|
|
data, err := os.ReadFile(pipeline)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
mustWriteTestFile(t, pipeline, string(data)+`scriptorium:
|
|
|
|
|
artifacts:
|
|
|
|
|
alpha: {enabled: true, prompt_id: alpha, output_path: artifacts/alpha.md}
|
|
|
|
|
beta: {enabled: true, prompt_id: beta, output_path: artifacts/beta.md}
|
|
|
|
|
`)
|
|
|
|
|
plan, err := BuildBoundedPlan("analyze", "analyze")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
seed := manifest.New("2026-05-03", time.Now().UTC())
|
|
|
|
|
seed.Campaign = "sample-campaign"
|
|
|
|
|
for _, name := range plan.PrefixNames() {
|
|
|
|
|
seed.MarkStageSucceeded(name, time.Now().UTC(), nil)
|
|
|
|
|
}
|
|
|
|
|
path := artifacts.SessionManifestPathForCampaign(root, seed.Campaign, seed.SessionID)
|
|
|
|
|
if err := (&manifest.LocalStore{}).Save(context.Background(), path, seed); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return pipeline, campaign, session
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestNativeProgressDoesNotAdvertiseUnsavedArtifact(t *testing.T) {
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
pipeline, campaign, session := writeArtifactProgressConfig(t, root)
|
|
|
|
|
cause := errors.New("artifact checkpoint unavailable")
|
|
|
|
|
store := ¤tAnalyzeCheckpointFailingStore{Store: &manifest.LocalStore{}, CurrentLimit: 1, Err: cause}
|
|
|
|
|
runner := &analyzeCheckpointRunner{}
|
|
|
|
|
app, err := newApplication(commandDependencies{}, func(ctx context.Context, cfg *config.Config, plan BoundedPlan, opts RunOptions) (*RunSummary, error) {
|
|
|
|
|
opts.Env = &Env{Scriptorium: runner, ManifestStore: store}
|
|
|
|
|
return executePlan(ctx, cfg, plan, opts)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
var out, diagnostic bytes.Buffer
|
|
|
|
|
outcome := app.Execute(context.Background(), []string{"run", "2026-05-03", "--stage", "analyze", "--config", pipeline, "--campaign-file", campaign, "--session-file", session, "--progress", "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostic})
|
|
|
|
|
if outcome.Status != 1 || !errors.Is(outcome.Cause, cause) || strings.Contains(diagnostic.String(), "artifact completed") {
|
|
|
|
|
t.Fatalf("unsaved completion: outcome=%#v diagnostics=%s", outcome, diagnostic.String())
|
|
|
|
|
}
|
|
|
|
|
if len(runner.Calls) != 1 || runner.Calls[0] != "alpha" {
|
|
|
|
|
t.Fatalf("unexpected adapter work=%v", runner.Calls)
|
|
|
|
|
}
|
|
|
|
|
var report runReport
|
|
|
|
|
if err := json.Unmarshal(out.Bytes(), &report); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if report.Complete || len(report.AvailableArtifacts) != 0 {
|
|
|
|
|
t.Fatalf("uncommitted output advertised: %#v", report)
|
|
|
|
|
}
|
|
|
|
|
}
|