Wire process identity cancellation and shared HTTP server lifecycle

This commit is contained in:
2026-11-20 04:55:27 +00:00
parent 41dc1f5517
commit 7b570ed49d
12 changed files with 380 additions and 11 deletions

View File

@@ -11,6 +11,7 @@ steps:
export GOTOOLCHAIN=local export GOTOOLCHAIN=local
version="$CI_COMMIT_TAG" version="$CI_COMMIT_TAG"
revision="$CI_COMMIT_SHA"
dist="dist" dist="dist"
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium" pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
notes="docs/releases/$version.md" notes="docs/releases/$version.md"
@@ -31,7 +32,7 @@ steps:
output="$dist/scriptorium-$version-$goos-$goarch$suffix" output="$dist/scriptorium-$version-$goos-$goarch$suffix"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \ CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
go build -trimpath -ldflags "-s -w" \ go build -trimpath -ldflags "-s -w -X main.version=$version -X main.revision=$revision" \
-o "$output" "$pkg" -o "$output" "$pkg"
} }

View File

@@ -1,11 +1,29 @@
package main package main
import ( import (
"context"
"io"
"os" "os"
"os/signal"
"syscall"
"gitea.maximumdirect.net/eric/scriptorium/internal/adapter/cli" "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/cli"
"go.maximumdirect.net/commandkit"
) )
var version = "dev"
var revision = ""
func main() { func main() {
os.Exit(cli.Run(os.Args[1:], os.Stdout, os.Stderr)) os.Exit(runWithSignals(os.Args[1:], os.Stdout, os.Stderr))
}
func runWithSignals(args []string, stdout, stderr io.Writer) int {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
return run(ctx, args, stdout, stderr)
}
func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return cli.Execute(ctx, commandkit.Identity{Name: "scriptorium", Version: version, Revision: revision}, args, stdout, stderr).Status
} }

View File

@@ -86,8 +86,9 @@ The source-boundary guards reject committed workspaces, vendor trees, and
module replacements. module replacements.
It then runs the complete suite once with the race detector and builds Linux It then runs the complete suite once with the race detector and builds Linux
amd64 and arm64 binaries using `CGO_ENABLED=0`, `-trimpath`, and amd64 and arm64 binaries using `CGO_ENABLED=0`, `-trimpath`, and linker flags
`-ldflags '-s -w'`. The native test run uses the host platform and enables cgo. `-s -w -X main.version=<version> -X main.revision=<candidate-commit>`.
The native test run uses the host platform and enables cgo.
Validation disables workspaces, uses read-only module mode, and disables module Validation disables workspaces, uses read-only module mode, and disables module
and toolchain downloads. Missing dependencies must be provisioned separately; and toolchain downloads. Missing dependencies must be provisioned separately;
fix the environment and rerun the command. No persistent validation cache is fix the environment and rerun the command. No persistent validation cache is
@@ -111,6 +112,8 @@ publishes:
The matching checked-in note supplies the hosted release body. The workflow The matching checked-in note supplies the hosted release body. The workflow
owns image selection, packaging, checksum generation, and upload configuration. owns image selection, packaging, checksum generation, and upload configuration.
Its binaries receive the tag and source commit as the same version/revision
metadata used by local candidate builds.
Its upload step skips existing files and does not overwrite the release. Its upload step skips existing files and does not overwrite the release.
Source-publication success does not attest to workflow success or asset Source-publication success does not attest to workflow success or asset
availability. availability.

View File

@@ -581,6 +581,27 @@ explanation identifies the failure. No exhaustive filesystem matrix.
## Stage 7: Wire Process Metadata, Cancellation, And Server Startup ## Stage 7: Wire Process Metadata, Cancellation, And Server Startup
**Status: Complete.** Entrypoint execution now uses an interrupt/SIGTERM context,
cleans signal registration before process exit, and returns the native finalized
status. Version/revision string variables support linker metadata; local and
hosted release builds supply the selected version and full source commit without
changing targets, toolchain, publication guards, or release-completion semantics.
`serve` loads resolved settings once, constructs one restricted reader and shared
engine, retains the HTTP handler/limits/header timeout, and closes listener and
active connections on cancellation. Startup uses native Info reporting; canceled
and unrelated server failures retain operational outcomes and typed causes.
Verified on Go 1.27.4: focused native-adapter tests/race checks and vet, existing
HTTP race regressions (including reader/report/capacity behavior), release-script
tests, and local/hosted shell syntax checks. Explicit readiness coordination
covers server startup, a file-backed HTTP run with its execution report, and
shutdown during active engine work. Timeout checks distinguish native operation
budgets, earlier parent cancellation/deadline, zero/negative budgets, and a
per-attempt timeout. Framework load failures retain their native diagnostics.
Entrypoint package tests and the linked version/revision binary check are
deferred until the legacy files are removed; include them in Stage 9's full
application verification. No release was published.
**Primary files:** `cmd/scriptorium/main.go`, CLI `serve` declaration, **Primary files:** `cmd/scriptorium/main.go`, CLI `serve` declaration,
`scripts/release.sh`, `.woodpecker/release.yml`, and lifecycle tests. `scripts/release.sh`, `.woodpecker/release.yml`, and lifecycle tests.

View File

@@ -22,9 +22,7 @@ func Execute(ctx context.Context, identity commandkit.Identity, args []string, s
func newApp(identity commandkit.Identity) (*commandkit.App, error) { func newApp(identity commandkit.Identity) (*commandkit.App, error) {
definitions := promptSuite() definitions := promptSuite()
declarations := []commandkit.Declaration{ declarations := []commandkit.Declaration{
commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: func(commandkit.Invocation, serveInput) commandkit.Outcome { commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Summary: "Serve the HTTP API.", Flags: serveFlags(), Run: serveHTTP}),
return commandkit.Failure("Server startup is not implemented.")
}}),
configurationCommand(), configurationCommand(),
commandkit.Completion("scriptorium"), commandkit.Completion("scriptorium"),
} }

View File

@@ -0,0 +1,235 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
func TestRunCancellationAndNativeTimeoutLayers(t *testing.T) {
for _, test := range []struct {
name, operation, attempt string
cancelParent, localTimeout bool
parentDeadline bool
}{
{"parent cancellation", "10s", "0", true, false, false},
{"operation timeout", "500ms", "0", false, true, false},
{"zero operation budget", "0", "0", true, false, false},
{"negative operation budget", "-1s", "0", true, false, false},
{"attempt timeout", "0", "1", false, false, false},
{"earlier parent deadline", "10s", "0", false, false, true},
} {
t.Run(test.name, func(t *testing.T) {
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
close(started)
select {
case <-r.Context().Done():
close(stopped)
case <-release:
}
}))
defer provider.Close()
defer close(release)
nativePromptFixture(t, provider.URL+"/v1")
var ctx context.Context
var cancel context.CancelFunc
if test.parentDeadline {
ctx, cancel = context.WithTimeout(context.Background(), 500*time.Millisecond)
} else {
ctx, cancel = context.WithCancel(context.Background())
}
defer cancel()
var output, diagnostics bytes.Buffer
finished := make(chan commandkit.Outcome, 1)
args := append(deliveryArgs(), "--run-timeout="+test.operation, "--timeout-seconds="+test.attempt)
go func() {
finished <- Execute(ctx, commandkit.Identity{Name: "scriptorium"}, args, &output, &diagnostics)
}()
select {
case <-started:
case outcome := <-finished:
t.Fatalf("ended before provider dispatch: %+v %q", outcome, diagnostics.String())
case <-time.After(5 * time.Second):
t.Fatal("provider did not start")
}
if test.cancelParent {
cancel()
}
var outcome commandkit.Outcome
select {
case outcome = <-finished:
case <-time.After(5 * time.Second):
t.Fatal("run did not stop")
}
if outcome.Status != 1 || output.Len() != 0 || diagnostics.Len() == 0 || errors.Is(outcome.Cause, promptkit.ErrOperationTimeout) != test.localTimeout {
t.Fatalf("timeout/cancellation: %+v %q %q", outcome, output.String(), diagnostics.String())
}
if test.cancelParent && !errors.Is(outcome.Cause, context.Canceled) {
t.Fatalf("parent cause lost: %+v", outcome)
}
if !test.cancelParent && !errors.Is(outcome.Cause, context.DeadlineExceeded) {
t.Fatalf("deadline cause lost: %+v", outcome)
}
if !test.cancelParent && !test.parentDeadline && ctx.Err() != nil {
t.Fatal("engine deadline changed caller context")
}
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("provider request was not canceled")
}
})
}
}
func TestServeStartupHTTPAndCancellation(t *testing.T) {
var block atomic.Bool
started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
if block.Load() {
close(started)
select {
case <-r.Context().Done():
close(stopped)
case <-release:
}
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"content":"served"}}]}`)
}))
defer provider.Close()
defer close(release)
nativePromptFixture(t, provider.URL+"/v1")
writeNativeFile(t, "artifacts/transcript.md", "file from restricted root")
ready := make(chan string, 1)
listen := func(ctx context.Context, network, addr string) (net.Listener, error) {
listener, err := (&net.ListenConfig{}).Listen(ctx, network, addr)
if err == nil {
ready <- listener.Addr().String()
}
return listener, err
}
app, err := commandkit.New(commandkit.Identity{Name: "scriptorium"}, commandkit.NewCommand(commandkit.CommandSpec[serveInput]{Name: "serve", Flags: serveFlags(), Run: func(inv commandkit.Invocation, input serveInput) commandkit.Outcome {
return serveWithListener(inv, input, listen)
}}))
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var output, diagnostics bytes.Buffer
finished := make(chan commandkit.Outcome, 1)
go func() {
finished <- app.Execute(ctx, []string{"serve", "--config=app.yml", "--addr=127.0.0.1:0", "--artifact-root=artifacts", "--max-request-bytes=0", "--max-artifact-bytes=0", "--max-response-bytes=0", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
}()
var addr string
select {
case addr = <-ready:
case outcome := <-finished:
t.Fatalf("startup: %+v %q", outcome, diagnostics.String())
case <-time.After(5 * time.Second):
t.Fatal("server did not listen")
}
client := &http.Client{Timeout: 5 * time.Second}
defer client.CloseIdleConnections()
const payload = `{"prompt_id":"support/summary","profile_id":"support/writer","session_id":"server-session","vars":{"mode":"","title":"Title"},"inputs":{"transcript":{"type":"file","uri":"transcript.md"},"unlisted":{"type":"inline","body":"literal"}},"model":{"timeout_seconds":0},"include_execution_report":true}`
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
if err != nil {
t.Fatal(err)
}
var value map[string]any
decodeErr := json.NewDecoder(response.Body).Decode(&value)
response.Body.Close()
if response.StatusCode != 200 || decodeErr != nil || value["execution"] == nil {
t.Fatalf("HTTP response: %d %+v %v", response.StatusCode, value, decodeErr)
}
metadata := value["metadata"].(map[string]any)
if metadata["selected_profile_id"] != "support/writer" || metadata["session_id"] != "server-session" {
t.Fatalf("HTTP contract changed: %+v", value)
}
block.Store(true)
requestFinished := make(chan error, 1)
go func() {
response, err := client.Post("http://"+addr+"/v1/runs", "application/json", strings.NewReader(payload))
if response != nil {
response.Body.Close()
}
requestFinished <- err
}()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("active HTTP request did not reach engine")
}
cancel()
select {
case outcome := <-finished:
if outcome.Status != 1 || !errors.Is(outcome.Cause, context.Canceled) || output.Len() != 0 || diagnostics.Len() == 0 {
t.Fatalf("shutdown: %+v %q %q", outcome, output.String(), diagnostics.String())
}
case <-time.After(5 * time.Second):
t.Fatal("HTTP server did not close")
}
select {
case <-requestFinished:
case <-time.After(5 * time.Second):
t.Fatal("active HTTP connection remained open")
}
select {
case <-stopped:
case <-time.After(5 * time.Second):
t.Fatal("active engine work was not canceled")
}
if connection, err := net.DialTimeout("tcp", addr, time.Second); err == nil {
connection.Close()
t.Fatal("listener remained open")
}
}
func TestServeListenAndConfigurationFailures(t *testing.T) {
nativePromptFixture(t, "http://127.0.0.1:1/v1")
for _, args := range [][]string{
{"serve", "--config=app.yml", "--addr=invalid:address:syntax"},
{"serve", "--config=absent.yml"},
} {
outcome, output, diagnostics := executeNative(t, args...)
if outcome.Status != 1 || outcome.Cause == nil || output != "" || diagnostics == "" {
t.Fatalf("serve failure: %+v %q %q", outcome, output, diagnostics)
}
}
writeNativeFile(t, "invalid.yml", "server:\n max_request_bytes: -1\n")
outcome, _, diagnostics := executeNative(t, "serve", "--config=invalid.yml")
if outcome.Status != 2 || !strings.Contains(diagnostics, "validate.error") {
t.Fatalf("serve configuration classification: %+v %q", outcome, diagnostics)
}
writeNativeFile(t, "bad-framework.yml", "profiles: [\n")
writeNativeFile(t, "bad-framework-app.yml", "promptkit:\n configuration: bad-framework.yml\n")
for _, command := range []string{"run", "serve"} {
args := []string{command}
if command == "run" {
args = append(args, "any")
}
args = append(args, "--config=bad-framework-app.yml")
outcome, _, diagnostics := executeNative(t, args...)
if outcome.Status != 2 || !strings.Contains(diagnostics, "PromptKit configuration") || !strings.Contains(diagnostics, "yaml.invalid") {
t.Fatalf("framework diagnostics: %+v %q", outcome, diagnostics)
}
}
}

View File

@@ -1,6 +1,7 @@
package cli package cli
import ( import (
"errors"
"text/template" "text/template"
"time" "time"
@@ -74,9 +75,13 @@ func requestFromRunInput(input runInput) promptkit.RunRequest {
func runPrompt(invocation commandkit.Invocation, input runInput) commandkit.Outcome { func runPrompt(invocation commandkit.Invocation, input runInput) commandkit.Outcome {
state := &acquisitionState{} state := &acquisitionState{}
engine, err := loadEngine(invocation.Context, input.Selection, applicationInput{}, state) engine, err := loadEngine(invocation.Context, input.Selection, applicationInput{}, state)
applicationFailed := errors.Is(err, errApplicationAcquisition)
acquired, err := attachApplicationEvidence(commandkit.Acquired[*promptkit.Engine]{Value: engine}, err, state, 0, false) acquired, err := attachApplicationEvidence(commandkit.Acquired[*promptkit.Engine]{Value: engine}, err, state, 0, false)
queueDiagnostics(invocation.Reporter(), acquired.Diagnostics) queueDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil { if err != nil {
if !applicationFailed {
return frameworkFailure(invocation.Reporter(), err, acquired.Explanation)
}
return requestFailure(err, acquired.Explanation) return requestFailure(err, acquired.Explanation)
} }
inspection, err := engine.InspectConfiguration() inspection, err := engine.InspectConfiguration()

View File

@@ -6,6 +6,7 @@ import (
"go.maximumdirect.net/ckconfig" "go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit" "go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit" "go.maximumdirect.net/promptkit"
) )
@@ -19,6 +20,16 @@ func completedRunOutcome(result *promptkit.RunResult) commandkit.Outcome {
return commandkit.Success() return commandkit.Success()
} }
func frameworkFailure(reporter *commandkit.Reporter, err error, explanation commandkit.Explanation) commandkit.Outcome {
var native *configkit.LoadError
if errors.As(err, &native) && native != nil {
diagnostics, terminal := ckconfig.AdaptDiagnostics("PromptKit configuration:\n", native.Sources(), native.Diagnostics())
queueDiagnostics(reporter, diagnostics)
explanation.Details = append(explanation.Details, terminal.Details...)
}
return requestFailure(err, explanation)
}
func requestFailure(err error, explanation commandkit.Explanation) commandkit.Outcome { func requestFailure(err error, explanation commandkit.Explanation) commandkit.Outcome {
message := "The requested operation failed." message := "The requested operation failed."
switch { switch {
@@ -27,7 +38,7 @@ func requestFailure(err error, explanation commandkit.Explanation) commandkit.Ou
case errors.Is(err, context.Canceled): case errors.Is(err, context.Canceled):
message = "The operation was canceled." message = "The operation was canceled."
case errors.Is(err, context.DeadlineExceeded): case errors.Is(err, context.DeadlineExceeded):
message = "The caller deadline expired." message = "A request deadline expired."
default: default:
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage { if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
return commandkit.Usage("Invalid configuration.", explanation.Details...).WithCause(err) return commandkit.Usage("Invalid configuration.", explanation.Details...).WithCause(err)

View File

@@ -83,7 +83,7 @@ func loadApplication(ctx context.Context, selection ckconfig.Selection, input ap
return nil, err return nil, err
} }
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return nil, err return result, err
} }
return result, nil return result, nil
} }

View File

@@ -0,0 +1,76 @@
package cli
import (
"context"
"errors"
"fmt"
"net"
"net/http"
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func serveHTTP(invocation commandkit.Invocation, input serveInput) commandkit.Outcome {
return serveWithListener(invocation, input, (&net.ListenConfig{}).Listen)
}
func serveWithListener(invocation commandkit.Invocation, input serveInput, listen func(context.Context, string, string) (net.Listener, error)) commandkit.Outcome {
result, err := loadApplication(invocation.Context, input.Selection, input.Application, struct{}{})
var sources []configkit.SourceInfo
var records []configkit.Diagnostic
if result != nil {
sources, records = result.Sources(), result.Diagnostics()
}
var native *configkit.LoadError
if errors.As(err, &native) && native != nil {
sources, records = native.Sources(), native.Diagnostics()
}
diagnostics, explanation := ckconfig.AdaptDiagnostics("Application configuration:\n", sources, records)
queueDiagnostics(invocation.Reporter(), diagnostics)
if err != nil {
return requestFailure(err, explanation)
}
settings := appconfig.Settings(result.Config)
reader, err := httpadapter.NewRestrictedArtifactReader(settings.ArtifactRoot, settings.MaxArtifactBytes)
if err != nil {
return commandkit.Failure("Could not initialize the HTTP artifact reader.").WithCause(err)
}
engine, err := newEngine(settings, promptkit.WithArtifactReader(reader))
if err != nil {
return frameworkFailure(invocation.Reporter(), err, explanation)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
return requestFailure(err, explanation)
}
diagnostics, explanation = ckconfig.AdaptDiagnostics("PromptKit configuration:\n", inspection.Sources(), inspection.Diagnostics())
queueDiagnostics(invocation.Reporter(), diagnostics)
server := &http.Server{
Addr: settings.ServerAddr,
Handler: httpadapter.NewHandlerWithOptions(engine, httpadapter.HandlerOptions{MaxRequestBytes: settings.MaxRequestBytes, MaxResponseBytes: settings.MaxResponseBytes}),
ReadHeaderTimeout: defaults.HTTPReadHeaderTimeoutDefault,
}
listener, err := listen(invocation.Context, "tcp", settings.ServerAddr)
if err != nil {
return commandkit.Failure("Could not listen for HTTP requests.").WithCause(err)
}
_ = invocation.Reporter().Info(fmt.Sprintf("Serving HTTP on %s.", listener.Addr()))
closed := make(chan struct{})
var cancellationCloseErr error
stop := context.AfterFunc(invocation.Context, func() { cancellationCloseErr = server.Close(); close(closed) })
serveErr := server.Serve(listener)
if !stop() {
<-closed
}
closeErr := errors.Join(cancellationCloseErr, server.Close())
if errors.Is(serveErr, http.ErrServerClosed) && invocation.Context.Err() != nil {
return commandkit.Failure("HTTP server stopped after cancellation.").WithCause(errors.Join(invocation.Context.Err(), closeErr))
}
return commandkit.Failure("HTTP server stopped unexpectedly.").WithCause(errors.Join(serveErr, closeErr))
}

View File

@@ -92,7 +92,8 @@ host_os=$(go env GOHOSTOS)
host_arch=$(go env GOHOSTARCH) host_arch=$(go env GOHOSTARCH)
GOOS="$host_os" GOARCH="$host_arch" CGO_ENABLED=1 go test -race ./... GOOS="$host_os" GOARCH="$host_arch" CGO_ENABLED=1 go test -race ./...
for arch in amd64 arm64; do for arch in amd64 arm64; do
GOOS=linux GOARCH="$arch" CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \ GOOS=linux GOARCH="$arch" CGO_ENABLED=0 go build -trimpath \
-ldflags "-s -w -X main.version=$version -X main.revision=$candidate" \
-o "$release_tmp/scriptorium-$version-linux-$arch" ./cmd/scriptorium -o "$release_tmp/scriptorium-$version-linux-$arch" ./cmd/scriptorium
done done

View File

@@ -210,7 +210,7 @@ func TestReleasePublishesOnlyValidatedTag(t *testing.T) {
t.Fatalf("unexpected validation calls: %s", log) t.Fatalf("unexpected validation calls: %s", log)
} }
for i, arch := range []string{"amd64", "arm64"} { for i, arch := range []string{"amd64", "arm64"} {
if !strings.HasPrefix(lines[i+1], "build -trimpath -ldflags -s -w -o ") || if !strings.HasPrefix(lines[i+1], "build -trimpath -ldflags -s -w -X main.version="+version+" -X main.revision="+f.candidate+" -o ") ||
!strings.Contains(lines[i+1], "scriptorium-"+version+"-linux-"+arch+" ./cmd/scriptorium|") || !strings.Contains(lines[i+1], "scriptorium-"+version+"-linux-"+arch+" ./cmd/scriptorium|") ||
!strings.HasSuffix(lines[i+1], "|off|-mod=readonly|off|off|local|linux|"+arch+"|0") { !strings.HasSuffix(lines[i+1], "|off|-mod=readonly|off|off|local|linux|"+arch+"|0") {
t.Fatalf("unexpected build: %s", lines[i+1]) t.Fatalf("unexpected build: %s", lines[i+1])