package cli import ( "bytes" "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "slices" "strings" "sync/atomic" "testing" "time" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) func deliveryArgs() []string { return []string{"run", "support/summary", "--config=app.yml", "--profile=support/writer", "--session-id=session", "--var=mode=", "--var=title=Title", "--input-text=transcript=text", "--input-text=unlisted=literal"} } func TestSeparateReportsUseNativeFormats(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"choices":[{"message":{"content":"artifact"}}]}`) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") for _, format := range []commandkit.Format{"", commandkit.FormatText, commandkit.FormatYAML, commandkit.FormatJSON} { path := "report-" + string(format) + ".different-extension" writeNativeFile(t, path, "old-report-must-be-replaced-completely") args := append(deliveryArgs(), "--format=json", "--report="+path) if format != "" { args = append(args, "--report-format="+string(format)) } outcome, output, diagnostics := executeNative(t, args...) if outcome.Status != 0 { t.Fatalf("%q: %+v %q", format, outcome, diagnostics) } var result promptkit.RunResult if err := json.Unmarshal([]byte(output), &result); err != nil { t.Fatal(err) } // Use the serialized native value rather than round-tripping timing // metadata through RunResult: a positive sub-millisecond duration is // serialized as zero but would be omitted after that typed round trip. var nativeValue any if err := json.Unmarshal([]byte(output), &nativeValue); err != nil { t.Fatal(err) } var expected bytes.Buffer if err := commandkit.RenderReport(&expected, commandkit.Report{Value: nativeValue}, format); err != nil { t.Fatal(err) } actual, err := os.ReadFile(path) if err != nil { t.Fatal(err) } if !bytes.Equal(actual, expected.Bytes()) || !bytes.Contains(actual, []byte(result.RunID)) || !bytes.Contains(actual, []byte("selected_profile_id")) { t.Fatalf("report %q differs from native renderer:\n%s\nexpected:\n%s", format, actual, expected.Bytes()) } } outcome, output, diagnostics := executeNative(t, append(deliveryArgs(), "--format=text", "--report=metadata.yaml", "--report-format=json")...) if outcome.Status != 0 || output != "artifact" { t.Fatalf("independent formats: %+v %q %q", outcome, output, diagnostics) } data, err := os.ReadFile("metadata.yaml") if err != nil { t.Fatal(err) } var result promptkit.RunResult if err := json.Unmarshal(data, &result); err != nil { t.Fatal(err) } if string(result.Artifact.Body) != "artifact" || result.RunID == "" { t.Fatalf("report lost full result: %+v", result) } outcome, output, diagnostics = executeNative(t, append(deliveryArgs(), "--quiet", "--report=-", "--report-format=json")...) if outcome.Status != 0 || output != "" { t.Fatalf("quiet report: %+v %q %q", outcome, output, diagnostics) } if data, err := os.ReadFile("-"); err != nil || !bytes.Contains(data, []byte("run_id")) { t.Fatalf("literal dash report: %s %v", data, err) } } func TestReportValidationAndResultlessFailureLeaveFilesUntouched(t *testing.T) { var calls atomic.Int64 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1) w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"choices":[{"message":{"content":"artifact"}}]}`) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") for _, flags := range [][]string{{"--report="}, {"--report=untouched", "--report-format=invalid"}, {"--report=untouched", "--report-format="}} { outcome, _, diagnostics := executeNative(t, append(deliveryArgs(), flags...)...) if outcome.Status != 2 || diagnostics == "" || calls.Load() != 0 { t.Fatalf("preflight %v: %+v %q calls=%d", flags, outcome, diagnostics, calls.Load()) } } if _, err := os.Stat("untouched"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("validation created file: %v", err) } writeNativeFile(t, "existing-report", "keep-existing") outcome, output, diagnostics := executeNative(t, "run", "missing", "--config=app.yml", "--report=existing-report") if outcome.Status != 1 || output != "" || diagnostics == "" { t.Fatalf("resultless failure: %+v %q %q", outcome, output, diagnostics) } if data, err := os.ReadFile("existing-report"); err != nil || string(data) != "keep-existing" { t.Fatalf("resultless failure touched report: %q %v", data, err) } outcome, output, diagnostics = executeNative(t, append(deliveryArgs(), "--report=missing-parent/report")...) if outcome.Status != 1 || output != "artifact" || !strings.Contains(diagnostics, "report") { t.Fatalf("report failure after main output: %+v %q %q", outcome, output, diagnostics) } if _, err := os.Stat("missing-parent"); !errors.Is(err, os.ErrNotExist) { t.Fatalf("created report parent: %v", err) } } type failedDeliveryWriter struct{ cause error } func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause } func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) { t.Chdir(t.TempDir()) result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}} stdoutFailure := errors.New("stdout failure") for _, invalid := range []bool{false, true} { result.Validation.Status = promptkit.ValidationPassed if invalid { result.Validation.Status = promptkit.ValidationFailed } for _, path := range []string{"report", "absent/report"} { app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: path}}, result) }})) if err != nil { t.Fatal(err) } var diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: failedDeliveryWriter{stdoutFailure}, Err: &diagnostics}) want := 1 if invalid { want = 4 } if outcome.Status != want || !errors.Is(outcome.Presentation.Output.WriteError, stdoutFailure) { t.Fatalf("delivery outcome: %+v %q", outcome, diagnostics.String()) } if invalid && (outcome.Kind != commandkit.OutcomeDomain || outcome.Domain != "validation_failed") { t.Fatalf("domain outcome lost: %+v", outcome) } if path == "report" { if data, err := os.ReadFile(path); err != nil || !bytes.Contains(data, []byte("known-result")) { t.Fatalf("report not attempted: %q %v", data, err) } } else { var fileError *os.PathError if !errors.As(outcome.Cause, &fileError) || !strings.Contains(diagnostics.String(), "report file") { t.Fatalf("report error lost: %+v %q", outcome, diagnostics.String()) } } } } // Unsupported result data exercises native rendering failure, not a local // replacement serializer. Both main and file delivery failures survive. result.Validation.Status = promptkit.ValidationPassed result.EffectiveModelParams.ExtraParams = map[string]any{"invalid": make(chan int)} app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "render-failed"}}, result) }})) if err != nil { t.Fatal(err) } var output, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "--format=json"}, commandkit.Streams{Out: &output, Err: &diagnostics}) if outcome.Status != 1 || outcome.Presentation.Output.RenderError == nil || outcome.Cause == nil || !strings.Contains(diagnostics.String(), "report file") { t.Fatalf("render errors lost: %+v %q", outcome, diagnostics.String()) } if _, err := os.Stat("render-failed"); err != nil { t.Fatalf("file effect not attempted: %v", err) } } func TestCLIProcessOutput(t *testing.T) { if os.Getenv("SCRIPTORIUM_CLI_CHILD") == "1" { separator := slices.Index(os.Args, "--") if separator < 0 { os.Exit(2) } outcome := Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, os.Args[separator+1:], os.Stdout, os.Stderr) os.Exit(outcome.Status) } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") fmt.Fprint(w, `{"choices":[{"message":{"content":"redirected artifact"}}]}`) })) defer server.Close() nativePromptFixture(t, server.URL+"/v1") executable, err := os.Executable() if err != nil { t.Fatal(err) } for _, quiet := range []bool{false, true} { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) args := append([]string{"-test.run=^TestCLIProcessOutput$", "--"}, deliveryArgs()...) args = append(args, "--report=process-report", "--report-format=json") if quiet { args = append(args, "--quiet") } command := exec.CommandContext(ctx, executable, args...) command.Env = append(os.Environ(), "SCRIPTORIUM_CLI_CHILD=1") path := filepath.Join(t.TempDir(), "redirected-output") file, err := os.Create(path) if err != nil { cancel() t.Fatal(err) } command.Stdout = file var diagnostics bytes.Buffer command.Stderr = &diagnostics runErr := command.Run() closeErr := file.Close() cancel() if runErr != nil || closeErr != nil { t.Fatalf("subprocess: %v %v %q", runErr, closeErr, diagnostics.String()) } data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } want := "redirected artifact" if quiet { want = "" } if string(data) != want { t.Fatalf("redirected quiet=%t: %q", quiet, data) } data, err = os.ReadFile("process-report") if err != nil || !bytes.Contains(data, []byte("run_id")) { t.Fatalf("independent process report: %s %v", data, err) } } }