2026-07-26 17:17:35 +00:00
package cli
import (
"bytes"
2026-11-22 04:05:44 +00:00
"context"
2026-07-26 17:17:35 +00:00
"encoding/json"
"errors"
2026-11-22 04:05:44 +00:00
"io"
2026-07-26 17:17:35 +00:00
"os"
"path/filepath"
"strings"
2026-11-22 04:05:44 +00:00
"sync"
2026-07-26 17:17:35 +00:00
"testing"
2026-11-22 04:05:44 +00:00
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
2026-07-26 17:17:35 +00:00
)
2026-11-22 04:05:44 +00:00
func decodeCompletionReport ( t * testing . T , format commandkit . Format , content [ ] byte ) map [ string ] any {
t . Helper ( )
if format == commandkit . FormatYAML {
value , err := commandkit . DecodeYAMLReport ( bytes . NewReader ( content ) )
if err != nil {
t . Fatalf ( "decode native YAML report: %v\n%s" , err , content )
}
content , err = json . Marshal ( value )
if err != nil {
t . Fatal ( err )
}
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
var report map [ string ] any
if err := json . Unmarshal ( content , & report ) ; err != nil {
t . Fatalf ( "decode native report: %v\n%s" , err , content )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
return report
}
func assertCompletedRunFiles ( t * testing . T , file string , captured bool ) {
t . Helper ( )
run := filepath . Join ( filepath . Dir ( file ) , "output-must-not-exist" , "run-1" )
if content , err := os . ReadFile ( filepath . Join ( run , "lanes" , "notes.json" ) ) ; err != nil || len ( content ) == 0 {
t . Fatalf ( "published domain artifact: %s %v" , content , err )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
files , err := os . ReadDir ( run )
if err != nil || len ( files ) != 1 || files [ 0 ] . Name ( ) != "lanes" {
t . Fatalf ( "unexpected reporting sidecars: %#v %v" , files , err )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
if captured {
debug := filepath . Join ( filepath . Dir ( file ) , "debug-must-not-exist" , "run-1" )
content , err := os . ReadFile ( filepath . Join ( debug , "outcome.json" ) )
if err != nil {
t . Fatal ( err )
}
var terminal struct {
Outcome struct {
Published bool ` json:"published" `
} ` json:"outcome" `
Error string ` json:"error" `
CaptureError string ` json:"capture_error" `
}
if err := json . Unmarshal ( content , & terminal ) ; err != nil || ! terminal . Outcome . Published || terminal . Error != "" || terminal . CaptureError != "" {
t . Fatalf ( "capture was not completed before reporting: %s %v" , content , err )
}
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
}
2026-07-26 17:17:35 +00:00
2026-11-22 04:05:44 +00:00
type completedRunWriter struct {
t * testing . T
file string
captured bool
content bytes . Buffer
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
func ( writer * completedRunWriter ) Write ( content [ ] byte ) ( int , error ) {
assertCompletedRunFiles ( writer . t , writer . file , writer . captured )
return writer . content . Write ( content )
}
2026-07-26 17:17:35 +00:00
2026-11-22 04:05:44 +00:00
func TestNativeRunCompletionFormatsFollowPublicationAndCapture ( t * testing . T ) {
for _ , format := range [ ] commandkit . Format { commandkit . FormatText , commandkit . FormatJSON , commandkit . FormatYAML } {
t . Run ( string ( format ) , func ( t * testing . T ) {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
output := & completedRunWriter { t : t , file : file , captured : true }
var diagnostics bytes . Buffer
outcome := app . Execute ( context . Background ( ) , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--debug" , "--format" , string ( format ) } , commandkit . Streams { Out : output , Err : & diagnostics } )
if outcome . Status != 0 || ! strings . Contains ( diagnostics . String ( ) , "Running pipeline" ) {
t . Fatalf ( "completion: %#v stderr=%s" , outcome , diagnostics . String ( ) )
}
directory := filepath . Join ( filepath . Dir ( file ) , "output-must-not-exist" , "run-1" )
debug := filepath . Join ( filepath . Dir ( file ) , "debug-must-not-exist" , "run-1" )
if format == commandkit . FormatText {
if ! strings . Contains ( output . content . String ( ) , "Success: demo produced 1 artifacts in " + directory ) || ! strings . Contains ( output . content . String ( ) , "Debug capture: " + debug ) || strings . Contains ( output . content . String ( ) , "fallback" ) {
t . Fatalf ( "native completion text: %s" , output . content . String ( ) )
}
} else {
report := decodeCompletionReport ( t , format , output . content . Bytes ( ) )
if len ( report ) != 4 || report [ "pipeline" ] != "demo" || report [ "artifact_count" ] != float64 ( 1 ) || report [ "output_directory" ] != directory || report [ "debug_directory" ] != debug {
t . Fatalf ( "native report: %#v" , report )
}
}
if probe . produced != 1 {
t . Fatalf ( "reporting reran a producer: %d" , probe . produced )
}
} )
2026-07-26 17:17:35 +00:00
}
}
2026-11-22 04:05:44 +00:00
func TestNativeRunDisclosesFallbackIndependentlyOfDiagnosticVisibility ( t * testing . T ) {
for _ , format := range [ ] commandkit . Format { commandkit . FormatText , commandkit . FormatJSON , commandkit . FormatYAML } {
for _ , verbose := range [ ] bool { false , true } {
probe := & runProbe { correctFirst : true }
app , file , input , _ := runFixture ( t , probe )
args := [ ] string { "pipeline" , "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--set" , ` /pipelines/demo/mode="permissive" ` , "--set" , "/pipelines/demo/max_additional_turns=0" , "--format" , string ( format ) }
if verbose {
args = append ( args , "--verbose" )
}
var output , diagnostics bytes . Buffer
outcome := app . Execute ( context . Background ( ) , args , commandkit . Streams { Out : & output , Err : & diagnostics } )
if outcome . Status != 0 || probe . produced != 1 {
t . Fatalf ( "permissive completion: %#v %s" , outcome , diagnostics . String ( ) )
}
if verbose && ! strings . Contains ( diagnostics . String ( ) , "WARN:" ) || ! verbose && diagnostics . Len ( ) != 0 {
t . Fatalf ( "native warning visibility: %s" , diagnostics . String ( ) )
}
if format == commandkit . FormatText {
if ! strings . Contains ( output . String ( ) , "Success with fallback:" ) || ! strings . Contains ( output . String ( ) , "1 operations used fallback." ) {
t . Fatalf ( "fallback text: %s" , output . String ( ) )
}
} else {
report := decodeCompletionReport ( t , format , output . Bytes ( ) )
if report [ "fallback_count" ] != float64 ( 1 ) || report [ "artifact_count" ] != float64 ( 1 ) || len ( report ) != 4 {
t . Fatalf ( "fallback report: %#v" , report )
}
}
assertCompletedRunFiles ( t , file , false )
}
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
}
2026-07-26 17:17:35 +00:00
2026-11-22 04:05:44 +00:00
func TestNativeRunQuietAndDebugUseNativeModeOrdering ( t * testing . T ) {
for _ , test := range [ ] struct {
flags [ ] string
quiet bool
progress bool
} {
{ [ ] string { "--debug" , "--quiet" } , true , false } ,
{ [ ] string { "--quiet" , "--debug" } , false , true } ,
{ [ ] string { "--debug" , "--progress=false" } , false , false } ,
} {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
var output , diagnostics bytes . Buffer
args := append ( [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--format" , "json" } , test . flags ... )
outcome := app . Execute ( context . Background ( ) , args , commandkit . Streams { Out : & output , Err : & diagnostics } )
if outcome . Status != 0 || test . quiet != ( output . Len ( ) == 0 ) || test . progress != strings . Contains ( diagnostics . String ( ) , "Running pipeline" ) {
t . Fatalf ( "native mode order: %#v output=%s stderr=%s" , outcome , output . String ( ) , diagnostics . String ( ) )
}
assertCompletedRunFiles ( t , file , true )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
}
2026-07-26 17:17:35 +00:00
2026-11-22 04:05:44 +00:00
func TestNativeRunCancellationKeepsItsCauseAndPublishesNothing ( t * testing . T ) {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
ctx , cancel := context . WithCancel ( context . Background ( ) )
defer cancel ( )
probe . onGenerate = cancel
var output , diagnostics bytes . Buffer
outcome := app . Execute ( ctx , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--debug" , "--quiet" } , commandkit . Streams { Out : & output , Err : & diagnostics } )
if outcome . Status != 1 || ! errors . Is ( outcome . Cause , context . Canceled ) || output . Len ( ) != 0 || diagnostics . Len ( ) == 0 {
t . Fatalf ( "cancellation: %#v stdout=%s stderr=%s" , outcome , output . String ( ) , diagnostics . String ( ) )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
if _ , err := os . Stat ( filepath . Join ( filepath . Dir ( file ) , "output-must-not-exist" ) ) ; ! os . IsNotExist ( err ) {
t . Fatalf ( "cancellation published artifacts: %v" , err )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
}
func TestNativeRunPrimaryFailureSurvivesCaptureAndDiagnosticFailures ( t * testing . T ) {
primary := errors . New ( "provider failure sentinel" )
delivery := errors . New ( "diagnostic failure sentinel" )
probe := & runProbe { providerErr : primary }
app , file , input , _ := runFixture ( t , probe )
debug := filepath . Join ( filepath . Dir ( file ) , "debug-must-not-exist" , "run-1" )
probe . onGenerate = func ( ) {
if err := os . Mkdir ( filepath . Join ( debug , "outcome.json" ) , 0700 ) ; err != nil {
t . Error ( err )
}
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
var output bytes . Buffer
outcome := app . Execute ( context . Background ( ) , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--debug" , "--llm-transient-retries=0" } , commandkit . Streams { Out : & output , Err : failingResultWriter { cause : delivery } } )
var captureFailure * os . LinkError
if outcome . Status != 1 || ! errors . Is ( outcome . Cause , primary ) || ! errors . As ( outcome . Cause , & captureFailure ) || errors . Is ( outcome . Cause , delivery ) || ! errors . Is ( outcome . Presentation . Diagnostics . WriteError , delivery ) || output . Len ( ) != 0 || len ( probe . requests ) != 1 {
t . Fatalf ( "primary/capture/delivery failure: %#v" , outcome )
}
if ! strings . Contains ( outcome . Explanation . Message , debug ) {
t . Fatalf ( "failed run capture location unavailable: %#v" , outcome )
2026-07-26 17:17:35 +00:00
}
}
2026-11-22 04:05:44 +00:00
func TestNativeRunReportDeliveryFailuresRetainArtifactsAndNativePresentation ( t * testing . T ) {
sentinel := errors . New ( "result delivery sentinel" )
for _ , partial := range [ ] bool { false , true } {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
var output io . Writer = failingResultWriter { cause : sentinel }
prefix := & partialResultWriter { cause : sentinel }
if partial {
output = prefix
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
var diagnostics bytes . Buffer
outcome := app . Execute ( context . Background ( ) , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--debug" , "--format" , "json" } , commandkit . Streams { Out : output , Err : & diagnostics } )
if outcome . Status != 1 || outcome . Cause != nil || ! errors . Is ( outcome . Presentation . Output . WriteError , sentinel ) || outcome . Presentation . Output . State != commandkit . DeliveryIncomplete || probe . produced != 1 || partial && prefix . content . Len ( ) == 0 {
t . Fatalf ( "delivery failure: %#v %s" , outcome , diagnostics . String ( ) )
}
assertCompletedRunFiles ( t , file , true )
for _ , root := range [ ] string { "output-must-not-exist" , "debug-must-not-exist" } {
if ! strings . Contains ( diagnostics . String ( ) , filepath . Join ( filepath . Dir ( file ) , root , "run-1" ) ) {
t . Fatalf ( "missing available path after delivery failure: %s" , diagnostics . String ( ) )
}
2026-07-26 17:17:35 +00:00
}
}
}
2026-11-22 04:05:44 +00:00
func TestNativeRunDiagnosticDeliveryFailureIsNotAnOperationFailure ( t * testing . T ) {
sentinel := errors . New ( "diagnostic delivery sentinel" )
probe := & runProbe { correctFirst : true }
app , file , input , _ := runFixture ( t , probe )
var output bytes . Buffer
outcome := app . Execute ( context . Background ( ) , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--set" , ` /pipelines/demo/mode="permissive" ` , "--set" , "/pipelines/demo/max_additional_turns=0" , "--format" , "json" , "--verbose" } , commandkit . Streams { Out : & output , Err : failingResultWriter { cause : sentinel } } )
if outcome . Status != 1 || outcome . Cause != nil || ! errors . Is ( outcome . Presentation . Diagnostics . WriteError , sentinel ) || outcome . Presentation . Diagnostics . State != commandkit . DeliveryIncomplete {
t . Fatalf ( "diagnostic delivery: %#v" , outcome )
}
if report := decodeCompletionReport ( t , commandkit . FormatJSON , output . Bytes ( ) ) ; report [ "fallback_count" ] != float64 ( 1 ) {
t . Fatalf ( "completed report was replaced by a delivery envelope: %#v" , report )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
if ! strings . Contains ( outcome . Explanation . Message , filepath . Join ( filepath . Dir ( file ) , "output-must-not-exist" , "run-1" ) ) {
t . Fatalf ( "presentation failure lost available artifacts: %#v" , outcome )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
assertCompletedRunFiles ( t , file , false )
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
func TestNativeRunCaptureFailureAfterPublicationDoesNotReportSuccess ( t * testing . T ) {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
var once sync . Once
debug := filepath . Join ( filepath . Dir ( file ) , "debug-must-not-exist" , "run-1" )
probe . onGenerate = func ( ) {
once . Do ( func ( ) {
// The external provider boundary runs after capture allocation but
// before terminalization. Make only the terminal file unusable.
if err := os . Mkdir ( filepath . Join ( debug , "outcome.json" ) , 0700 ) ; err != nil {
t . Fatal ( err )
}
} )
}
var output , diagnostics bytes . Buffer
outcome := app . Execute ( context . Background ( ) , [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--debug" , "--format" , "json" } , commandkit . Streams { Out : & output , Err : & diagnostics } )
if outcome . Status != 1 || outcome . Cause == nil || output . Len ( ) != 0 || probe . produced != 1 {
t . Fatalf ( "post-publication capture failure: %#v stdout=%s stderr=%s" , outcome , output . String ( ) , diagnostics . String ( ) )
}
assertCompletedRunFiles ( t , file , false )
if ! strings . Contains ( diagnostics . String ( ) , debug ) || ! strings . Contains ( diagnostics . String ( ) , filepath . Join ( filepath . Dir ( file ) , "output-must-not-exist" , "run-1" ) ) {
t . Fatalf ( "post-publication locations: %s" , diagnostics . String ( ) )
}
if _ , err := os . Stat ( filepath . Join ( debug , "executions.json" ) ) ; err != nil {
t . Fatalf ( "existing capture was removed: %v" , err )
}
2026-07-26 17:17:35 +00:00
}
2026-11-22 04:05:44 +00:00
func TestNativeRunFailureCategoriesPreserveCausesWithoutSuccessReports ( t * testing . T ) {
for _ , test := range [ ] struct {
name string
configure func ( * testing . T , * runProbe , string , string ) [ ] string
status int
cause error
} {
{ "missing configuration" , func ( t * testing . T , _ * runProbe , _ , input string ) [ ] string {
return [ ] string { "run" , "demo" , "--input-file" , input , "--config" , filepath . Join ( t . TempDir ( ) , "missing.yml" ) }
} , 1 , nil } ,
{ "invalid configuration" , func ( t * testing . T , _ * runProbe , file , input string ) [ ] string {
return [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--set" , "/version=5" }
} , 2 , nil } ,
{ "unknown pipeline" , func ( _ * testing . T , _ * runProbe , file , input string ) [ ] string {
return [ ] string { "run" , "missing" , "--config" , file , "--input-file" , input }
} , 2 , nil } ,
{ "missing input" , func ( t * testing . T , _ * runProbe , file , _ string ) [ ] string {
return [ ] string { "run" , "demo" , "--config" , file , "--input-file" , filepath . Join ( t . TempDir ( ) , "missing.txt" ) }
} , 1 , os . ErrNotExist } ,
{ "publication failure" , func ( t * testing . T , _ * runProbe , file , input string ) [ ] string {
path := filepath . Join ( t . TempDir ( ) , "occupied" )
if err := os . WriteFile ( path , [ ] byte ( "keep" ) , 0600 ) ; err != nil {
t . Fatal ( err )
}
return [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--output-dir" , path }
} , 1 , nil } ,
{ "provider failure" , func ( _ * testing . T , probe * runProbe , file , input string ) [ ] string {
probe . providerErr = io . ErrUnexpectedEOF
return [ ] string { "run" , "demo" , "--config" , file , "--input-file" , input , "--cache" , "bypass" , "--llm-transient-retries=0" }
} , 1 , io . ErrUnexpectedEOF } ,
} {
t . Run ( test . name , func ( t * testing . T ) {
probe := & runProbe { }
app , file , input , _ := runFixture ( t , probe )
var output , diagnostics bytes . Buffer
args := append ( test . configure ( t , probe , file , input ) , "--quiet" , "--format" , "json" )
outcome := app . Execute ( context . Background ( ) , args , commandkit . Streams { Out : & output , Err : & diagnostics } )
if outcome . Status != test . status || test . cause != nil && ! errors . Is ( outcome . Cause , test . cause ) || output . Len ( ) != 0 || diagnostics . Len ( ) == 0 {
t . Fatalf ( "failure category: %#v stdout=%s stderr=%s" , outcome , output . String ( ) , diagnostics . String ( ) )
}
if test . name == "missing configuration" || test . name == "invalid configuration" {
var load * configkit . LoadError
if ! errors . As ( outcome . Cause , & load ) {
t . Fatalf ( "lost native load cause: %#v" , outcome )
}
}
} )
2026-07-26 17:17:35 +00:00
}
}