Report completed runs through native presentation and preserve failure outcomes
This commit is contained in:
@@ -2,21 +2,34 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.maximumdirect.net/commandkit"
|
||||
)
|
||||
|
||||
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
func buildNativeExecutable(t *testing.T) string {
|
||||
t.Helper()
|
||||
binary := filepath.Join(t.TempDir(), "notarius")
|
||||
build := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-o", binary, "../../cmd/notarius")
|
||||
build.Env = append(os.Environ(), "GOWORK=off")
|
||||
if output, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build executable: %v\n%s", err, output)
|
||||
}
|
||||
return binary
|
||||
}
|
||||
|
||||
func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
binary := buildNativeExecutable(t)
|
||||
for _, route := range [][]string{{"run"}, {"pipeline", "run"}} {
|
||||
help := exec.Command(binary, append(append([]string(nil), route...), "--help")...)
|
||||
output, err := help.CombinedOutput()
|
||||
@@ -31,3 +44,68 @@ func TestExecutableRunRoutesDeclareTheSameRequiredInputs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutableReportsDomainArtifactsAndQualifiedFallbacks(t *testing.T) {
|
||||
binary := buildNativeExecutable(t)
|
||||
for _, fallback := range []bool{false, true} {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if fallback {
|
||||
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(writer).Encode(map[string]any{"id": "local-fake", "object": "chat.completion", "choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": `{"spell_casts":[]}`}, "finish_reason": "stop"}}})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
for _, format := range []commandkit.Format{commandkit.FormatText, commandkit.FormatJSON, commandkit.FormatYAML} {
|
||||
t.Run(fmt.Sprintf("%s/fallback=%t", format, fallback), func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
configuration := fmt.Sprintf(`version: 6
|
||||
output: {directory: output}
|
||||
pipelines:
|
||||
demo:
|
||||
input: {module: seriatim}
|
||||
chunk: {module: generic}
|
||||
llm_profile: local-fake
|
||||
mode: permissive
|
||||
max_additional_turns: 0
|
||||
artifacts:
|
||||
spells:
|
||||
extract: {module: dnd/spells}
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: local-fake
|
||||
endpoint: %q
|
||||
model: fake-model
|
||||
api_key_required: false
|
||||
`, server.URL+"/v1")
|
||||
file := filepath.Join(root, "config.yml")
|
||||
input := filepath.Join(root, "input.json")
|
||||
for path, content := range map[string]string{file: configuration, input: `{"metadata":{"id":"report-source"},"segments":[{"id":1,"start":0,"end":1,"speaker":"GM","text":"The party rests."}]}`} {
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
command := exec.Command(binary, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-transient-retries=0", "--format", string(format), "--verbose")
|
||||
var output, diagnostics bytes.Buffer
|
||||
command.Stdout, command.Stderr = &output, &diagnostics
|
||||
if err := command.Run(); err != nil {
|
||||
t.Fatalf("subprocess: %v stdout=%s stderr=%s", err, output.String(), diagnostics.String())
|
||||
}
|
||||
if format == commandkit.FormatText {
|
||||
if !strings.Contains(output.String(), "produced 1 artifacts") || fallback != strings.Contains(output.String(), "Success with fallback:") {
|
||||
t.Fatalf("completion text: %s", output.String())
|
||||
}
|
||||
} else {
|
||||
report := decodeCompletionReport(t, format, output.Bytes())
|
||||
if report["artifact_count"] != float64(1) || report["pipeline"] != "demo" || fallback && report["fallback_count"] != float64(1) || !fallback && report["fallback_count"] != nil {
|
||||
t.Fatalf("completion report: %#v", report)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(report["output_directory"].(string), "lanes", "spells.json")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user