From 1f2f102dd51aa5a1258ec337639cd7b0083c799f Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:19:10 -0800 Subject: [PATCH 1/8] Drain the native self-host before stopping workerd --- apps/hosted/self-host/native/main.go | 266 +++--- apps/hosted/self-host/native/shutdown.go | 420 +++++++++ apps/hosted/self-host/native/shutdown_test.go | 822 ++++++++++++++++++ 3 files changed, 1351 insertions(+), 157 deletions(-) create mode 100644 apps/hosted/self-host/native/shutdown.go create mode 100644 apps/hosted/self-host/native/shutdown_test.go diff --git a/apps/hosted/self-host/native/main.go b/apps/hosted/self-host/native/main.go index 60d492b9d2..77a596a41d 100644 --- a/apps/hosted/self-host/native/main.go +++ b/apps/hosted/self-host/native/main.go @@ -564,8 +564,17 @@ func serve(mode string) error { } defer listener.Close() server := &http.Server{Handler: &bridge{config: values, repositories: repositories, temporary: temporary}, ReadHeaderTimeout: 10 * time.Second} - defer server.Close() - serverError := make(chan error, 1) + closeBridge := func() { server.Close() } + if mode == "serve" { + base, cancel := context.WithCancel(context.Background()) + server.BaseContext = func(net.Listener) context.Context { return base } + closeBridge = func() { + cancel() + server.Close() + } + } + defer closeBridge() + serverError := make(chan error, 2) go func() { serverError <- server.Serve(listener) }() config, err := os.ReadFile(filepath.Join(runtime, "workerd.capnp")) if err != nil { @@ -731,11 +740,17 @@ func serve(mode string) error { // images used are refused after the upgrade; apps get new ones on their next invocation. // Only the product process gets it; the Motel collector never sees it. handleSecret := "EXECUTOR_CREDENTIAL_HANDLE_SECRET=" + credentialHandleSecret(values["EXECUTOR_ENCRYPTION_KEY"]) - stopApps, restartApps := func() {}, func() {} + var signals chan os.Signal + if mode == "serve" { + signals = make(chan os.Signal, 2) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(signals) + } + var apps *supervisor if separate { // Only the app process seals handles, so only it gets the secret. It starts first; product // calls to apps fail as disconnected until it listens. - stopApps, restartApps = superviseProcess("App process", func() *exec.Cmd { + apps = startSupervisor("App process", func() *exec.Cmd { // A process that exited leaves its socket file behind; the next one binds the same path. os.Remove(appsSocket) appArgs := []string{"serve", appsConfigPath, "-I/", "--experimental", "--socket-addr=apps=unix:" + appsSocket, "--external-addr=product-self=unix:" + selfSocket, "--external-addr=product-callbacks=unix:" + callbacksSocket} @@ -746,24 +761,22 @@ func serve(mode string) error { apps.Env = append(apps.Env, handleSecret) return apps }, time.Second, 30*time.Second, time.After) - defer stopApps() + defer stopChildren(workerdGrace, time.After, apps) } else { command.Env = append(command.Env, handleSecret) } if err = command.Start(); err != nil { - stopApps() return err } - stopped := make(chan error, 1) - go func() { stopped <- command.Wait() }() + product := waitProcess("Product process", command) if mode == "export" { defer func() { command.Process.Signal(syscall.SIGTERM) select { - case <-stopped: + case <-product.exited: case <-time.After(15 * time.Second): command.Process.Kill() - <-stopped + <-product.exited } }() if len(os.Args) != 3 { @@ -774,11 +787,11 @@ func serve(mode string) error { forget, err := recordTelemetrySocket(directory, collector) if err != nil { command.Process.Kill() - <-stopped + <-product.exited return err } defer forget() - stopCollector := supervise(func() *exec.Cmd { + telemetryCollector := startSupervisor("Telemetry collector", func() *exec.Cmd { // A collector that was killed leaves its socket file behind, and the collector never // removes a path it did not create; this directory is the host's own, so the next one // binds the same path. @@ -792,88 +805,84 @@ func serve(mode string) error { command.Dir = runtime return command }, time.Second, 30*time.Second, time.After) - defer stopCollector() - - proxy := productProxy(filepath.Join(temporary, "product.sock"), workerdIdleTimeout, proxies...) - if trustedProxy != nil { - proxy = productProxyFor(filepath.Join(temporary, "product.sock"), workerdIdleTimeout, trustedProxy.clientIP) - } publicListener, err := net.Listen("tcp", net.JoinHostPort(setting("HOST", "0.0.0.0"), port)) if err != nil { command.Process.Kill() - <-stopped + <-product.exited + if apps != nil { + stopChildren(workerdGrace, time.After, apps) + } + stopChildren(collectorGrace, time.After, telemetryCollector) return err } + socketPath := filepath.Join(temporary, "product.sock") + edge := newEdge() + addressOf := func(request *http.Request) string { + return clientAddress(request.RemoteAddr, request.Header.Values("X-Forwarded-For"), proxies) + } + if trustedProxy != nil { + addressOf = trustedProxy.clientIP + } + proxy := productProxyFor(socketPath, workerdIdleTimeout, addressOf, edge) opened := make(chan struct{}) - public := &http.Server{Handler: startingGate(opened, publicHandler(proxy)), ReadHeaderTimeout: 10 * time.Second} - defer public.Close() + public := &http.Server{ + Handler: edge.handler(startingGate(opened, publicHandler(proxy))), + BaseContext: func(net.Listener) context.Context { return edge.base }, + ReadHeaderTimeout: 10 * time.Second, + } go func() { serverError <- public.Serve(publicListener) }() // One app isolate in a synchronous loop holds workerd's only JavaScript thread, so the product // and every other app stop answering and no timeout inside workerd can fire. OSS workerd does // not enforce CPU limits, so the host bounds the outage instead: once the product has answered, // a run of failed health probes kills workerd, this host exits non-zero, and the container's // restart policy starts a fresh process. - watchdogDone := make(chan struct{}) - defer close(watchdogDone) + watchCtx, endWatching := context.WithCancel(context.Background()) + var watchers sync.WaitGroup + stalled := make(chan error, 1) if interval, failures, enabled := watchdogSettings(); enabled { - probe := productProbe(filepath.Join(temporary, "product.sock"), min(interval, 5*time.Second)) - go watchdog(probe, interval, failures, time.After, watchdogDone, func() { - fmt.Fprintf(os.Stderr, "Executor watchdog: the product did not answer %d health checks in a row (every %s); workerd is unresponsive, restarting it\n", failures, interval) - command.Process.Kill() + probe := healthProbe(watchCtx, socketPath, min(interval, 5*time.Second), false) + watchers.Go(func() { + watchdog(probe, interval, failures, time.After, watchCtx.Done(), func() { + stalled <- fmt.Errorf("watchdog: the product did not answer %d health checks in a row (every %s); workerd is unresponsive", failures, interval) + }) }) } // The app process has its own watchdog. Any answer from its socket shows its thread is free, // even a refusal. After a run of silent probes it restarts that process alone, then re-arms // once the new process answers. - if interval, failures, enabled := appWatchdogSettings(); separate && enabled { - probe := anyAnswer(appsSocket, min(interval, 5*time.Second)) - go func() { + if interval, failures, enabled := appWatchdogSettings(); apps != nil && enabled { + probe := healthProbe(watchCtx, appsSocket, min(interval, 5*time.Second), true) + watchers.Go(func() { for { stalled := false - watchdog(probe, interval, failures, time.After, watchdogDone, func() { + watchdog(probe, interval, failures, time.After, watchCtx.Done(), func() { stalled = true - fmt.Fprintf(os.Stderr, "Executor watchdog: the app process did not answer %d health checks in a row (every %s); restarting it, the product keeps running\n", failures, interval) - restartApps() + apps.restartUnlessStopping(watchCtx, func() { + fmt.Fprintf(os.Stderr, "Executor watchdog: the app process did not answer %d health checks in a row (every %s); restarting it, the product keeps running\n", failures, interval) + }) }) if !stalled { return } } - }() + }) } - stopAnnouncing := make(chan struct{}) - defer close(stopAnnouncing) - go announceReady(filepath.Join(temporary, "product.sock"), values["BETTER_AUTH_URL"], startupProgressInterval, time.Second, os.Stderr, opened, stopAnnouncing) - - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) - defer signal.Stop(signals) - select { - case err := <-stopped: - return err - case <-signals: - case err := <-serverError: - if !errors.Is(err, http.ErrServerClosed) { - fmt.Fprintln(os.Stderr, "Native host stopped") - } + announceCtx, stopAnnouncing := context.WithCancel(context.Background()) + watchers.Go(func() { + announceReadyContext(announceCtx, socketPath, values["BETTER_AUTH_URL"], startupProgressInterval, time.Second, os.Stderr, opened) + }) + parts := shutdownParts{ + signals: signals, serverError: serverError, stalled: stalled, + endWatching: endWatching, endAnnouncing: stopAnnouncing, joinWatching: watchers.Wait, + public: public, listener: publicListener, edge: edge, + closeIdle: proxy.Transport.(*http.Transport).CloseIdleConnections, closeBridge: closeBridge, + product: product, productExited: product.exited, collector: telemetryCollector, } - // Stop both processes together so shutdown fits one deadline, and so a collector - // already stopped by a terminal's process-group signal is not restarted meanwhile. - telemetryStopped := make(chan struct{}) - go func() { - stopCollector() - close(telemetryStopped) - }() - command.Process.Signal(syscall.SIGTERM) - select { - case err = <-stopped: - case <-time.After(15 * time.Second): - command.Process.Kill() - err = <-stopped + if apps != nil { + parts.apps = apps } - <-telemetryStopped - return err + return shutdown(parts, shutdownPlan{drainGrace, workerdGrace, collectorGrace, time.After, os.Stderr}) } // collectorBounds are the bounds the image gives its telemetry collector, the ones it gave Motel: @@ -890,8 +899,6 @@ const startupProgressInterval = 15 * time.Second // to 30 seconds, and logs every progress interval while it waits. It closes opened when the // product first answers, ready or not, and returns once the product is ready or done closes. func announceReady(socket, origin string, progress, retry time.Duration, log io.Writer, opened chan<- struct{}, done <-chan struct{}) { - started := time.Now() - fmt.Fprintln(log, "Executor is starting. The first start initializes the database, which can take a minute or more on slower hardware.") ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { @@ -901,6 +908,12 @@ func announceReady(socket, origin string, progress, retry time.Duration, log io. case <-ctx.Done(): } }() + announceReadyContext(ctx, socket, origin, progress, retry, log, opened) +} + +func announceReadyContext(ctx context.Context, socket, origin string, progress, retry time.Duration, log io.Writer, opened chan<- struct{}) { + started := time.Now() + fmt.Fprintln(log, "Executor is starting. The first start initializes the database, which can take a minute or more on slower hardware.") // A fresh connection each time: a pooled one could reach workerd as it closes it. client := &http.Client{Transport: &http.Transport{ DisableKeepAlives: true, @@ -910,10 +923,12 @@ func announceReady(socket, origin string, progress, retry time.Duration, log io. }} ticker := time.NewTicker(progress) defer ticker.Stop() + var requests sync.WaitGroup + defer requests.Wait() var answered sync.Once ready := func() <-chan bool { result := make(chan bool, 1) - go func() { + requests.Go(func() { request, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://product.internal/health", nil) request.Header.Set("x-executor-client-ip", "127.0.0.1") response, err := client.Do(request) @@ -924,14 +939,14 @@ func announceReady(socket, origin string, progress, retry time.Duration, log io. response.Body.Close() answered.Do(func() { close(opened) }) result <- response.StatusCode == http.StatusOK - }() + }) return result } answer := ready() var again <-chan time.Time for { select { - case <-done: + case <-ctx.Done(): return case <-ticker.C: fmt.Fprintf(log, "Executor is still starting (%s)\n", time.Since(started).Round(time.Second)) @@ -1068,12 +1083,11 @@ func (c *workerdConn) Close() error { func productProxy(socket string, upstreamIdleTimeout time.Duration, trusted ...netip.Prefix) *httputil.ReverseProxy { return productProxyFor(socket, upstreamIdleTimeout, func(request *http.Request) string { return clientAddress(request.RemoteAddr, request.Header.Values("X-Forwarded-For"), trusted) - }) + }, nil) } -func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf func(*http.Request) string) *httputil.ReverseProxy { - - return &httputil.ReverseProxy{ +func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf func(*http.Request) string, edge *edge) *httputil.ReverseProxy { + proxy := &httputil.ReverseProxy{ Rewrite: func(request *httputil.ProxyRequest) { request.SetURL(&url.URL{Scheme: "http", Host: "product.internal"}) request.Out.Host = request.In.Host @@ -1097,6 +1111,10 @@ func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf }, FlushInterval: -1, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(context.Cause(r.Context()), errStopping) { + stoppingResponse(w) + return + } // Until workerd listens, its socket is missing or refuses connections. if errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.ECONNREFUSED) { http.Error(w, "Executor is starting", http.StatusServiceUnavailable) @@ -1108,6 +1126,10 @@ func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf http.Error(w, "Executor did not complete the request", http.StatusBadGateway) }, } + if edge != nil { + proxy.ModifyResponse = edge.response + } + return proxy } // watchdogSettings reads EXECUTOR_WATCHDOG_INTERVAL (seconds, default 10) and @@ -1144,32 +1166,14 @@ func appWatchdogSettings() (time.Duration, int, bool) { return time.Duration(seconds * float64(time.Second)), failures, true } -// anyAnswer reports whether a process answered any HTTP response on its socket within timeout. -// Its JavaScript thread is free whenever it answers, even when it refuses the request. -func anyAnswer(socket string, timeout time.Duration) func() bool { - client := http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, "unix", socket) - }, - DisableKeepAlives: true, - }, - } - return func() bool { - response, err := client.Get("http://apps/health") - if err != nil { - return false - } - io.Copy(io.Discard, response.Body) - response.Body.Close() - return true - } -} - // productProbe asks the product's own socket for /health, bypassing the public proxy, and // reports whether it answered 200 within timeout. func productProbe(socket string, timeout time.Duration) func() bool { + return healthProbe(context.Background(), socket, timeout, false) +} + +// Any response from apps shows that its JavaScript thread is free, even a refusal. +func healthProbe(ctx context.Context, socket string, timeout time.Duration, anyAnswer bool) func() bool { client := http.Client{ Timeout: timeout, Transport: &http.Transport{ @@ -1179,14 +1183,19 @@ func productProbe(socket string, timeout time.Duration) func() bool { DisableKeepAlives: true, }, } + address := "http://product/health" + if anyAnswer { + address = "http://apps/health" + } return func() bool { - response, err := client.Get("http://product/health") + request, _ := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) + response, err := client.Do(request) if err != nil { return false } io.Copy(io.Discard, response.Body) response.Body.Close() - return response.StatusCode == http.StatusOK + return anyAnswer || response.StatusCode == http.StatusOK } } @@ -1232,65 +1241,8 @@ func supervise(newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, af // starts each wait. It returns a function that stops the process and waits for it to exit, and one // that kills the running process so it restarts. func superviseProcess(name string, newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) (func(), func()) { - done := make(chan struct{}) - finished := make(chan struct{}) - var mutex sync.Mutex - var current *exec.Cmd - go func() { - defer close(finished) - delay := minDelay - for { - command := newCommand() - started := time.Now() - if err := command.Start(); err != nil { - fmt.Fprintln(os.Stderr, name+" could not start:", err) - } else { - mutex.Lock() - current = command - mutex.Unlock() - exited := make(chan struct{}) - go func() { command.Wait(); close(exited) }() - select { - case <-exited: - fmt.Fprintln(os.Stderr, name+" stopped; restarting") - case <-done: - command.Process.Signal(syscall.SIGTERM) - select { - case <-exited: - case <-time.After(15 * time.Second): - command.Process.Kill() - <-exited - } - return - } - mutex.Lock() - current = nil - mutex.Unlock() - } - if time.Since(started) > maxDelay { - delay = minDelay - } - select { - case <-after(delay): - case <-done: - return - } - delay = min(delay*2, maxDelay) - } - }() - var once sync.Once - stop := func() { - once.Do(func() { close(done) }) - <-finished - } - kill := func() { - mutex.Lock() - defer mutex.Unlock() - if current != nil && current.Process != nil { - current.Process.Kill() - } - } - return stop, kill + s := startSupervisor(name, newCommand, minDelay, maxDelay, after) + return func() { stopChildren(workerdGrace, time.After, s) }, s.restart } // healthTimeout is how long `executor-host health` waits for /health. Every response, /health diff --git a/apps/hosted/self-host/native/shutdown.go b/apps/hosted/self-host/native/shutdown.go new file mode 100644 index 0000000000..f08dfb6cdf --- /dev/null +++ b/apps/hosted/self-host/native/shutdown.go @@ -0,0 +1,420 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "os/exec" + "sync" + "syscall" + "time" +) + +const ( + drainGrace = 10 * time.Second + workerdGrace = 10 * time.Second + collectorGrace = 3 * time.Second + // Let cancelled handlers release their upstreams and flush a 503 before Close tears + // down their downstreams. A client that isn't reading cannot extend this wait. + cutGrace = time.Second +) + +var errStopping = errors.New("Executor is stopping") + +func stoppingResponse(w http.ResponseWriter) { + w.Header().Set("Retry-After", "5") + http.Error(w, errStopping.Error(), http.StatusServiceUnavailable) +} + +type edge struct { + base context.Context + cut context.CancelCauseFunc + + mu sync.Mutex + active map[*edgeRequest]struct{} + idle chan struct{} +} + +type edgeRequestKey struct{} + +type edgeRequest struct { + method string + media string +} + +func newEdge() *edge { + base, cut := context.WithCancelCause(context.Background()) + idle := make(chan struct{}) + close(idle) + return &edge{base: base, cut: cut, active: make(map[*edgeRequest]struct{}), idle: idle} +} + +func (e *edge) handler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + request := &edgeRequest{method: r.Method, media: "pending"} + e.mu.Lock() + if errors.Is(context.Cause(e.base), errStopping) { + e.mu.Unlock() + stoppingResponse(w) + return + } + if len(e.active) == 0 { + e.idle = make(chan struct{}) + } + e.active[request] = struct{}{} + e.mu.Unlock() + defer func() { + e.mu.Lock() + delete(e.active, request) + if len(e.active) == 0 { + close(e.idle) + } + e.mu.Unlock() + }() + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), edgeRequestKey{}, request))) + }) +} + +func (e *edge) response(r *http.Response) error { + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if r.StatusCode == http.StatusSwitchingProtocols { + media = "upgrade" + } else { + if media == "" { + media = "unspecified" + } + r.Body = &cutBody{ReadCloser: r.Body, ctx: r.Request.Context()} + } + if request, ok := r.Request.Context().Value(edgeRequestKey{}).(*edgeRequest); ok { + e.mu.Lock() + request.media = media + e.mu.Unlock() + } + return nil +} + +type cutBody struct { + io.ReadCloser + ctx context.Context +} + +func (b *cutBody) Read(buf []byte) (int, error) { + n, err := b.ReadCloser.Read(buf) + if err != nil && errors.Is(context.Cause(b.ctx), errStopping) { + // ReverseProxy still aborts the response, but does not log an intentional cut + // as an upstream read error or present the truncated body as complete. + err = context.Canceled + } + return n, err +} + +func (e *edge) cutRequests(log io.Writer) <-chan struct{} { + e.mu.Lock() + counts := make(map[string]int) + for r := range e.active { + counts[r.method+" "+r.media]++ + } + e.cut(errStopping) + idle := e.idle + e.mu.Unlock() + fmt.Fprintf(log, "Executor shutdown: requests at cut: %v\n", counts) + return idle +} + +type process struct { + name string + cmd *exec.Cmd + exited chan struct{} + err error // published by closing exited +} + +func waitProcess(name string, cmd *exec.Cmd) *process { + p := &process{name: name, cmd: cmd, exited: make(chan struct{})} + go func() { + p.err = cmd.Wait() + close(p.exited) + }() + return p +} + +// stopProcess sends SIGTERM once, escalates only at the shared deadline, and reaps even +// after escalation. Only the process owner calls it. +func stopProcess(p *process, deadline <-chan struct{}) (killed bool) { + if p == nil { + return false + } + select { + case <-p.exited: + return false + default: + } + p.cmd.Process.Signal(syscall.SIGTERM) + select { + case <-p.exited: + return false + case <-deadline: + select { + case <-p.exited: + return false + default: + } + killed = p.cmd.Process.Kill() == nil + <-p.exited + return killed + } +} + +type stopper interface { + stop(time.Duration, <-chan struct{}) error +} + +type stoppable interface { + stopper + hold() +} + +func (p *process) stop(grace time.Duration, deadline <-chan struct{}) error { + if stopProcess(p, deadline) { + return killedChild(p.name, grace) + } + return p.err +} + +func killedChild(name string, grace time.Duration) error { + return fmt.Errorf("%s did not stop within %s; killed", name, grace) +} + +type shutdownParts struct { + signals <-chan os.Signal + serverError <-chan error + stalled <-chan error + endWatching func() + endAnnouncing func() + joinWatching func() + public publicServer + listener io.Closer + edge *edge + closeIdle func() + closeBridge func() + product stopper + productExited <-chan struct{} + apps stoppable + collector stoppable +} + +type publicServer interface { + Shutdown(context.Context) error + Close() error +} + +type shutdownPlan struct { + drain, workerd, collector time.Duration + after func(time.Duration) <-chan time.Time + log io.Writer +} + +// shutdown owns the host lifecycle. Its phases only move forward: +// serving -> draining -> stoppingWorkerd -> stoppingCollector -> stopped. +func shutdown(parts shutdownParts, plan shutdownPlan) error { + // Serving. Only this select can act on a product watchdog report. + var reason error + skipDrain := false + select { + case <-parts.signals: + case reason = <-parts.serverError: + case <-parts.productExited: + skipDrain = true + case reason = <-parts.stalled: + skipDrain = true + } + + // Draining. Stop creating upstream work before waiting on existing requests. + parts.endWatching() + parts.endAnnouncing() + if parts.apps != nil { + parts.apps.hold() + } + parts.collector.hold() + // Close explicitly as Serve may not yet have registered the listener. + parts.listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + drained := make(chan struct{}) + go func() { + parts.public.Shutdown(ctx) + close(drained) + }() + if !skipDrain { + select { + case <-drained: + case <-plan.after(plan.drain): + case <-parts.signals: + case <-parts.productExited: + } + } + idle := parts.edge.cutRequests(plan.log) + parts.closeIdle() + cutDeadline := plan.after(cutGrace) + select { + case <-drained: // regular handlers and their final response bytes + // Shutdown does not wait for hijacked handlers. + select { + case <-idle: + case <-cutDeadline: + } + case <-cutDeadline: + } + parts.public.Close() + parts.closeIdle() + cancel() + <-drained + + // Stopping workerd. Both processes share one deadline; neither waits on the + // other to receive SIGTERM. Keep the bridge and collector alive for both. + err := errors.Join(reason, stopChildren(plan.workerd, plan.after, parts.product, parts.apps)) + parts.closeBridge() + parts.joinWatching() + + // Stopping collector, then stopped. + return errors.Join(err, stopChildren(plan.collector, plan.after, parts.collector)) +} + +func stopChildren(grace time.Duration, after func(time.Duration) <-chan time.Time, children ...stopper) error { + deadline := make(chan struct{}) + timer := after(grace) + results := make(chan error, len(children)) + pending := 0 + for _, child := range children { + if child != nil { + pending++ + go func() { results <- child.stop(grace, deadline) }() + } + } + var result error + for pending > 0 { + select { + case err := <-results: + result = errors.Join(result, err) + pending-- + case <-timer: + close(deadline) + timer = nil + } + } + return result +} + +type childState uint8 + +const ( + running childState = iota + held + stopping + exited + killed +) + +type supervisor struct { + name string + mu sync.Mutex + state childState + current *process + held chan struct{} + finished chan struct{} + stopOnce sync.Once + stopErr error +} + +func startSupervisor(name string, newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) *supervisor { + s := &supervisor{name: name, held: make(chan struct{}), finished: make(chan struct{})} + go s.run(newCommand, minDelay, maxDelay, after) + return s +} + +func (s *supervisor) run(newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) { + defer close(s.finished) + delay := minDelay + for { + s.mu.Lock() + if s.state != running { + s.mu.Unlock() + return + } + cmd := newCommand() + started := time.Now() + err := cmd.Start() + var p *process + if err == nil { + p = waitProcess(s.name, cmd) + s.current = p + } + s.mu.Unlock() + if err != nil { + fmt.Fprintln(os.Stderr, s.name+" could not start:", err) + } else { + <-p.exited + s.mu.Lock() + s.current = nil + retry := s.state == running + s.mu.Unlock() + if !retry { + return + } + fmt.Fprintln(os.Stderr, s.name+" stopped; restarting") + } + if time.Since(started) > maxDelay { + delay = minDelay + } + select { + case <-after(delay): + case <-s.held: + return + } + delay = min(delay*2, maxDelay) + } +} + +func (s *supervisor) hold() { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == running { + s.state = held + close(s.held) + } +} + +func (s *supervisor) stop(grace time.Duration, deadline <-chan struct{}) error { + s.stopOnce.Do(func() { + s.hold() + s.mu.Lock() + s.state = stopping + p := s.current + s.mu.Unlock() + wasKilled := stopProcess(p, deadline) + <-s.finished + s.mu.Lock() + defer s.mu.Unlock() + s.state = exited + if wasKilled { + s.state = killed + s.stopErr = killedChild(s.name, grace) + } + }) + return s.stopErr +} + +func (s *supervisor) restart() { + s.restartUnlessStopping(context.Background(), func() {}) +} + +func (s *supervisor) restartUnlessStopping(ctx context.Context, report func()) { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == running && ctx.Err() == nil && s.current != nil { + report() + s.current.cmd.Process.Kill() + } +} diff --git a/apps/hosted/self-host/native/shutdown_test.go b/apps/hosted/self-host/native/shutdown_test.go new file mode 100644 index 0000000000..74f8519748 --- /dev/null +++ b/apps/hosted/self-host/native/shutdown_test.go @@ -0,0 +1,822 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +func TestMain(m *testing.M) { + if role := os.Getenv("EXECUTOR_TEST_CHILD"); role != "" { + runShutdownChild(role, os.Args[1], os.Args[2], os.Args[3], os.Args[4]) + os.Exit(0) + } + os.Exit(m.Run()) +} + +func journalEvent(path, event string) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + panic(err) + } + defer file.Close() + if _, err := fmt.Fprintln(file, event); err != nil { + panic(err) + } +} + +func runShutdownChild(role, name, journal, socket, edgeAddress string) { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGTERM) + quit := make(chan struct{}) + var requests sync.WaitGroup + var admissions sync.Mutex + accepting := true + var listener net.Listener + notify := func(event string) { + journalEvent(journal, name+":"+event) + fmt.Println(event) + } + if role == "drains" { + var err error + listener, err = net.Listen("unix", socket) + if err != nil { + panic(err) + } + server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/finish" { + close(quit) + return + } + admissions.Lock() + if !accepting { + admissions.Unlock() + return + } + requests.Add(1) + admissions.Unlock() + defer requests.Done() + defer notify("request-ended") + switch r.URL.Path { + case "/upgrade": + conn, rw, err := http.NewResponseController(w).Hijack() + if err != nil { + panic(err) + } + defer conn.Close() + fmt.Fprint(rw, "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: test\r\n\r\n") + rw.Flush() + io.Copy(conn, rw) + case "/short", "/pending": + notify("request-started") + select { + case <-quit: + fmt.Fprint(w, "complete") + case <-r.Context().Done(): + } + default: + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + fmt.Fprint(w, "data: open\n\n") + w.(http.Flusher).Flush() + <-r.Context().Done() + } + })} + go server.Serve(listener) + } + notify("ready") + if role == "dead" { + os.Exit(42) + } + <-signals + if edgeAddress != "" { + if conn, err := net.Dial("tcp", edgeAddress); err == nil { + conn.Close() + notify("edge-open") + } + } + notify("term") + if role == "stubborn" { + for range signals { + } + } + if listener != nil { + listener.Close() + admissions.Lock() + accepting = false + admissions.Unlock() + requests.Wait() + } + notify("exit") +} + +func await[T any](t *testing.T, ch <-chan T) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(3 * time.Second): + t.Fatal("timed out waiting for shutdown event") + var zero T + return zero + } +} + +type manualTimers struct { + mu sync.Mutex + timers map[time.Duration]chan time.Time +} + +// Distinguish the two equal production durations in the fake clock. +const testDrain = drainGrace + time.Millisecond + +func (m *manualTimers) after(delay time.Duration) <-chan time.Time { return m.timer(delay) } + +func (m *manualTimers) timer(delay time.Duration) chan time.Time { + m.mu.Lock() + defer m.mu.Unlock() + ch := m.timers[delay] + if ch == nil { + ch = make(chan time.Time, 1) + m.timers[delay] = ch + } + return ch +} + +type observedListener struct { + net.Listener + accepted chan struct{} +} + +func (l observedListener) Accept() (net.Conn, error) { + select { + case <-l.accepted: + default: + close(l.accepted) + } + return l.Listener.Accept() +} + +type observedChild struct { + *process + name string + events chan string +} + +func (p *observedChild) hold() {} + +func (p *observedChild) stop(grace time.Duration, deadline <-chan struct{}) error { + if p.name == "product" { + return p.process.stop(grace, deadline) + } + if stopProcess(p.process, deadline) { + return killedChild(p.name, grace) + } + return nil +} + +type shutdownFixture struct { + t *testing.T + parts shutdownParts + signals chan os.Signal + stalled chan error + timers manualTimers + log bytes.Buffer + journal, socket, url string + draining chan struct{} + children []*observedChild +} + +func newShutdownFixture(t *testing.T) *shutdownFixture { + t.Helper() + dir, err := os.MkdirTemp("", "executor-stop-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + f := &shutdownFixture{ + t: t, journal: filepath.Join(dir, "journal"), socket: filepath.Join(dir, "workerd.sock"), + signals: make(chan os.Signal, 2), stalled: make(chan error, 1), draining: make(chan struct{}), + timers: manualTimers{timers: make(map[time.Duration]chan time.Time)}, + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + edge := newEdge() + proxy := productProxyFor(f.socket, workerdIdleTimeout, func(*http.Request) string { return "127.0.0.1" }, edge) + public := &http.Server{ + Handler: edge.handler(publicHandler(proxy)), + BaseContext: func(net.Listener) context.Context { return edge.base }, + } + accepted := make(chan struct{}) + go public.Serve(observedListener{listener, accepted}) + await(t, accepted) + t.Cleanup(func() { + edge.cut(errStopping) + public.Close() + listener.Close() + }) + f.url = "http://" + listener.Addr().String() + f.parts = shutdownParts{ + signals: f.signals, stalled: f.stalled, public: public, listener: listener, edge: edge, + endWatching: func() { close(f.draining) }, endAnnouncing: func() {}, joinWatching: func() {}, + closeIdle: func() { + proxy.Transport.(*http.Transport).CloseIdleConnections() + if errors.Is(context.Cause(edge.base), errStopping) { + journalEvent(f.journal, "edge:cut") + } + }, + closeBridge: func() { journalEvent(f.journal, "bridge:close") }, + } + return f +} + +func testChildCommand(role, name, journal, socket, edge string) *exec.Cmd { + cmd := exec.Command(os.Args[0], name, journal, socket, edge) + cmd.Env = append(os.Environ(), "EXECUTOR_TEST_CHILD="+role, "GORACE=atexit_sleep_ms=0") + cmd.Stderr = os.Stderr + return cmd +} + +func (f *shutdownFixture) child(role, name string) *observedChild { + f.t.Helper() + cmd := testChildCommand(role, name, f.journal, f.socket, strings.TrimPrefix(f.url, "http://")) + // Use an explicit pipe so Wait does not close the read side before the final journal event. + read, write, err := os.Pipe() + if err != nil { + f.t.Fatal(err) + } + cmd.Stdout = write + if err := cmd.Start(); err != nil { + read.Close() + write.Close() + f.t.Fatal(err) + } + write.Close() + p := &observedChild{process: waitProcess(name, cmd), name: name, events: make(chan string, 16)} + go func() { + defer read.Close() + defer close(p.events) + scanner := bufio.NewScanner(read) + for scanner.Scan() { + p.events <- scanner.Text() + } + }() + f.t.Cleanup(func() { + cmd.Process.Kill() + await(f.t, p.exited) + }) + if got := await(f.t, p.events); got != "ready" { + f.t.Fatalf("%s first event = %q; want ready", name, got) + } + f.children = append(f.children, p) + return p +} + +func (f *shutdownFixture) start() <-chan error { + for _, child := range f.children { + switch child.name { + case "product": + f.parts.product, f.parts.productExited = child, child.exited + case "apps": + f.parts.apps = child + case "collector": + f.parts.collector = child + } + } + result := make(chan error, 1) + go func() { + result <- shutdown(f.parts, shutdownPlan{testDrain, workerdGrace, collectorGrace, f.timers.after, &f.log}) + }() + return result +} + +func (f *shutdownFixture) request(method, path string) *http.Response { + f.t.Helper() + result := await(f.t, f.beginRequest(method, path)) + if result.err != nil { + f.t.Fatal(result.err) + } + f.t.Cleanup(func() { result.response.Body.Close() }) + return result.response +} + +type requestResult struct { + response *http.Response + err error +} + +func (f *shutdownFixture) beginRequest(method, path string) <-chan requestResult { + req, err := http.NewRequest(method, f.url+path, nil) + if err != nil { + f.t.Fatal(err) + } + done := make(chan requestResult, 1) + go func() { + client := &http.Client{Timeout: 4 * time.Second} + response, err := client.Do(req) + done <- requestResult{response, err} + }() + return done +} + +func (f *shutdownFixture) journalOrder(pairs ...[2]string) { + f.t.Helper() + data, err := os.ReadFile(f.journal) + if err != nil { + f.t.Fatal(err) + } + log := string(data) + if strings.Contains(log, ":edge-open") { + f.t.Fatalf("listener still open when a child got SIGTERM:\n%s", log) + } + for _, pair := range pairs { + first, second := strings.Index(log, pair[0]+"\n"), strings.Index(log, pair[1]+"\n") + if first < 0 || second < 0 || first >= second { + f.t.Fatalf("want %s before %s:\n%s", pair[0], pair[1], log) + } + } +} + +func (f *shutdownFixture) clean(result <-chan error) { + f.t.Helper() + if err := await(f.t, result); err != nil { + f.t.Fatal(err) + } +} + +func TestStopClosesTheEdgeBeforeAskingWorkerdToExit(t *testing.T) { + f := newShutdownFixture(t) + f.child("drains", "product") + f.child("quick", "apps") + f.child("quick", "collector") + response := f.request(http.MethodPost, "/stream") + result := f.start() + f.signals <- syscall.SIGTERM + await(t, f.draining) + f.timers.timer(testDrain) <- time.Now() + f.clean(result) + response.Body.Close() + f.journalOrder( + [2]string{"edge:cut", "product:term"}, [2]string{"edge:cut", "apps:term"}, + [2]string{"product:request-ended", "product:exit"}, + [2]string{"product:exit", "bridge:close"}, [2]string{"apps:exit", "bridge:close"}, + [2]string{"bridge:close", "collector:term"}, + ) +} + +func TestStopLetsAShortRequestFinish(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("drains", "product") + f.child("quick", "collector") + response := f.beginRequest(http.MethodGet, "/short") + if got := await(t, product.events); got != "request-started" { + t.Fatal(got) + } + result := f.start() + f.signals <- syscall.SIGTERM + await(t, f.draining) + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", f.socket) + }} + defer transport.CloseIdleConnections() + release, err := (&http.Client{Transport: transport, Timeout: 3 * time.Second}).Get("http://workerd/finish") + if err != nil { + t.Fatal(err) + } + release.Body.Close() + reply := await(t, response) + if reply.err != nil { + t.Fatal(reply.err) + } + defer reply.response.Body.Close() + body, err := io.ReadAll(reply.response.Body) + if err != nil || string(body) != "complete" { + t.Fatalf("got %q, %v; want complete response", body, err) + } + f.clean(result) + f.journalOrder([2]string{"product:request-ended", "product:term"}) +} + +func TestStopCutsAStreamThatNeverEndsAtTheDrainDeadline(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("drains", "product") + f.child("quick", "collector") + response := f.request(http.MethodPost, "/private?secret=not-logged") + result := f.start() + f.signals <- syscall.SIGTERM + await(t, f.draining) + stream := bufio.NewReader(response.Body) + if line, err := stream.ReadString('\n'); err != nil || line != "data: open\n" { + t.Fatalf("stream during drain: %q, %v", line, err) + } + select { + case got := <-product.events: + t.Fatalf("product got %q before the drain timer", got) + default: + } + f.timers.timer(testDrain) <- time.Now() + if _, err := io.ReadAll(stream); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("cut response should be incomplete, got %v", err) + } + f.clean(result) + if log := f.log.String(); !strings.Contains(log, "POST text/event-stream:1") || strings.Contains(log, "secret") || strings.Contains(log, "private") { + t.Fatalf("unexpected cut log: %s", log) + } +} + +func TestSecondSignalEndsTheDrain(t *testing.T) { + f := newShutdownFixture(t) + f.child("drains", "product") + f.child("quick", "collector") + f.request(http.MethodPost, "/stream") + result := f.start() + f.signals <- syscall.SIGTERM + await(t, f.draining) + f.signals <- syscall.SIGINT + f.clean(result) +} + +func TestARequestCutWhileWaitingForHeadersIsAnswered503(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("drains", "product") + f.child("quick", "collector") + response := f.beginRequest(http.MethodGet, "/pending") + if got := await(t, product.events); got != "request-started" { + t.Fatal(got) + } + result := f.start() + f.signals <- syscall.SIGTERM + f.timers.timer(testDrain) <- time.Now() + reply := await(t, response) + if reply.err != nil { + t.Fatal(reply.err) + } + defer reply.response.Body.Close() + if reply.response.StatusCode != 503 || reply.response.Header.Get("Retry-After") == "" { + t.Fatalf("got %d %v; want 503 with Retry-After", reply.response.StatusCode, reply.response.Header) + } + f.clean(result) +} + +func openTestTunnel(t *testing.T, address string) (*bufio.Reader, net.Conn) { + t.Helper() + conn, err := net.DialTimeout("tcp", address, 3*time.Second) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conn.SetDeadline(time.Now().Add(4 * time.Second)) + fmt.Fprint(conn, "GET /upgrade HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: test\r\n\r\n") + reader := bufio.NewReader(conn) + response, err := http.ReadResponse(reader, nil) + if err != nil || response.StatusCode != 101 { + t.Fatalf("expected upgrade: %v %v", response, err) + } + fmt.Fprint(conn, "ping\n") + if line, err := reader.ReadString('\n'); line != "ping\n" || err != nil { + t.Fatalf("tunnel echo was %q, %v", line, err) + } + return reader, conn +} + +func TestStopCutsAHijackedTunnelAtTheDeadline(t *testing.T) { + f := newShutdownFixture(t) + f.child("drains", "product") + f.child("quick", "collector") + // A POST keeps the drain open even though http.Server ignores the hijacked tunnel. + f.request(http.MethodPost, "/stream") + tunnel, _ := openTestTunnel(t, strings.TrimPrefix(f.url, "http://")) + result := f.start() + f.signals <- syscall.SIGTERM + f.timers.timer(testDrain) <- time.Now() + if _, err := tunnel.ReadByte(); !errors.Is(err, io.EOF) { + t.Fatalf("tunnel still open after cut: %v", err) + } + f.clean(result) +} + +func TestSupervisorDoesNotRestartAfterHold(t *testing.T) { + for _, inBackoff := range []bool{false, true} { + t.Run(fmt.Sprintf("backoff=%t", inBackoff), func(t *testing.T) { + journal := filepath.Join(t.TempDir(), "child") + var starts atomic.Int32 + ready := make(chan struct{}, 1) + backoff := make(chan chan time.Time, 1) + s := startSupervisor("test", func() *exec.Cmd { + starts.Add(1) + cmd := testChildCommand("quick", "apps", journal, "", "") + cmd.Stdout = readyWriter{ready} + return cmd + }, time.Second, time.Second, func(time.Duration) <-chan time.Time { + timer := make(chan time.Time, 1) + backoff <- timer + return timer + }) + t.Cleanup(func() { stopChildren(time.Second, time.After, s) }) + await(t, ready) + s.mu.Lock() + child := s.current + s.mu.Unlock() + if inBackoff { + child.cmd.Process.Signal(syscall.SIGTERM) + timer := await(t, backoff) + s.hold() + timer <- time.Now() + } else { + s.hold() + reported := false + s.restartUnlessStopping(context.Background(), func() { reported = true }) + if reported { + t.Fatal("restart acted after hold") + } + child.cmd.Process.Signal(syscall.SIGTERM) + } + await(t, s.finished) + if starts.Load() != 1 || child.err != nil { + t.Fatalf("held child: starts = %d, exit = %v", starts.Load(), child.err) + } + }) + } +} + +type readyWriter struct{ ready chan<- struct{} } + +func (w readyWriter) Write(p []byte) (int, error) { + if bytes.Contains(p, []byte("ready\n")) { + w.ready <- struct{}{} + } + return len(p), nil +} + +func TestStopKillsOnlyAChildThatIgnoresSIGTERM(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("drains", "product") + apps := f.child("stubborn", "apps") + collector := f.child("quick", "collector") + result := f.start() + f.signals <- syscall.SIGTERM + if got := await(t, apps.events); got != "term" { + t.Fatal(got) + } + await(t, product.exited) + for _, child := range []*observedChild{apps, collector} { + select { + case <-child.exited: + t.Fatalf("%s exited before apps grace expired", child.name) + default: + } + } + f.timers.timer(workerdGrace) <- time.Now() + if err := await(t, result); err == nil || err.Error() != "apps did not stop within 10s; killed" { + t.Fatalf("unexpected result: %v", err) + } + if !apps.cmd.ProcessState.Sys().(syscall.WaitStatus).Signaled() || apps.cmd.ProcessState.Sys().(syscall.WaitStatus).Signal() != syscall.SIGKILL || collector.err != nil || product.err != nil { + t.Fatalf("exit statuses: apps %v, collector %v, product %v", apps.err, collector.err, product.err) + } + f.journalOrder([2]string{"product:exit", "bridge:close"}, [2]string{"bridge:close", "collector:term"}) +} + +func TestStopAfterTheProductExitedSkipsTheDrain(t *testing.T) { + f := newShutdownFixture(t) + // A blocked handler makes waiting for graceful HTTP Shutdown observable. + enter, leave := make(chan struct{}), make(chan struct{}) + existing := f.parts.edge.handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(enter) + defer close(leave) + <-r.Context().Done() + })) + server := &http.Server{Handler: existing, BaseContext: func(net.Listener) context.Context { return f.parts.edge.base }} + f.parts.public.Close() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f.parts.public, f.parts.listener, f.url = server, listener, "http://"+listener.Addr().String() + go server.Serve(listener) + t.Cleanup(func() { server.Close(); listener.Close() }) + f.beginRequest(http.MethodGet, "/") + await(t, enter) + product := f.child("dead", "product") + f.child("quick", "collector") + await(t, product.exited) + result := f.start() + await(t, leave) + var exited *exec.ExitError + if err := await(t, result); !errors.As(err, &exited) || exited.ExitCode() != 42 { + t.Fatalf("want product exit 42, got %v", err) + } +} + +func TestStopFindsChildrenAlreadyGone(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("quick", "product") + for _, name := range []string{"apps", "collector"} { + await(t, f.child("dead", name).exited) + } + result := f.start() + f.signals <- syscall.SIGTERM + f.clean(result) + if product.err != nil { + t.Fatal(product.err) + } + // No timer was fired. An already-dead child must not use up a grace period. +} + +func TestAStalledProductAfterTheStopBeganIsIgnored(t *testing.T) { + f := newShutdownFixture(t) + f.child("drains", "product") + f.child("quick", "collector") + f.request(http.MethodPost, "/stream") + result := f.start() + f.signals <- syscall.SIGTERM + await(t, f.draining) + f.stalled <- errors.New("stale watchdog report") + f.signals <- syscall.SIGTERM + f.clean(result) + if len(f.stalled) != 1 { + t.Fatal("drain consumed a stale watchdog report") + } +} + +type closeFunc func() + +func (f closeFunc) Close() error { f(); return nil } + +type modelPublic struct{ closeFunc } + +func (modelPublic) Shutdown(ctx context.Context) error { <-ctx.Done(); return ctx.Err() } + +type modelChild struct { + name string + log func(string) + exited chan struct{} + err error + term chan struct{} +} + +func (c *modelChild) hold() { c.log("hold " + c.name) } + +func (c *modelChild) stop(grace time.Duration, deadline <-chan struct{}) error { + defer c.log("reap " + c.name) + select { + case <-c.exited: + return c.err + default: + } + c.log("term " + c.name) + close(c.term) + select { + case <-c.exited: + return c.err + case <-deadline: + select { + case <-c.exited: + return c.err + default: + } + c.log("kill " + c.name) + return killedChild(c.name, grace) + } +} + +func TestShutdownKeepsItsOrderForEveryEventOrder(t *testing.T) { + const ( + secondSignal = iota + productExit + appsExit + collectorExit + drainTimeout + workerdTimeout + ) + events := []int{secondSignal, productExit, appsExit, collectorExit, drainTimeout, workerdTimeout} + var permutations func(int) + permutations = func(from int) { + if from < len(events) { + for i := from; i < len(events); i++ { + events[from], events[i] = events[i], events[from] + permutations(from + 1) + events[from], events[i] = events[i], events[from] + } + return + } + t.Run(fmt.Sprint(events), func(t *testing.T) { + var mu sync.Mutex + var trace []string + log := func(event string) { + mu.Lock() + defer mu.Unlock() + trace = append(trace, event) + } + snapshot := func() []string { + mu.Lock() + defer mu.Unlock() + return slices.Clone(trace) + } + child := func(name string) *modelChild { + return &modelChild{name, log, make(chan struct{}), nil, make(chan struct{})} + } + product, apps, collector := child("product"), child("apps"), child("collector") + product.err = errors.New("product failed") + public := modelPublic{func() { log("close public") }} + signals := make(chan os.Signal, 2) + timers := manualTimers{timers: make(map[time.Duration]chan time.Time)} + timers.timer(cutGrace) <- time.Now() + e := newEdge() + stopped := make(chan error, 1) + draining := make(chan struct{}) + go func() { + stopped <- shutdown(shutdownParts{ + signals: signals, edge: e, public: public, product: product, productExited: product.exited, + apps: apps, collector: collector, listener: closeFunc(func() { + log("close listener") + close(draining) + }), + endWatching: func() { log("watchdogs off") }, + endAnnouncing: func() { log("announcer off") }, + joinWatching: func() {}, + closeIdle: func() { + if errors.Is(context.Cause(e.base), errStopping) { + log("cut") + } + }, + closeBridge: func() { log("close bridge") }, + }, shutdownPlan{testDrain, workerdGrace, collectorGrace, timers.after, io.Discard}) + }() + signals <- syscall.SIGTERM + await(t, draining) + cutAllowed, killAllowed := false, false + check := func(trace []string) { + t.Helper() + before := func(first, second string) { + i, j := slices.Index(trace, first), slices.Index(trace, second) + if j >= 0 && (i < 0 || i >= j) { + t.Fatalf("%s must precede %s: %v", first, second, trace) + } + } + for _, name := range []string{"product", "apps"} { + for _, prerequisite := range []string{"hold apps", "hold collector", "watchdogs off", "announcer off", "close listener", "cut", "close public"} { + before(prerequisite, "term "+name) + } + before("reap "+name, "close bridge") + before("term "+name, "kill "+name) + if !killAllowed && slices.Contains(trace, "kill "+name) { + t.Fatalf("child killed before its grace: %v", trace) + } + } + before("close bridge", "term collector") + if !cutAllowed && slices.Contains(trace, "cut") { + t.Fatalf("premature cut: %v", trace) + } + if slices.Contains(trace, "kill collector") { + t.Fatalf("collector used the workerd deadline: %v", trace) + } + } + for _, event := range events { + switch event { + case secondSignal: + cutAllowed = true + signals <- syscall.SIGINT + case productExit: + cutAllowed = true + close(product.exited) + case appsExit: + close(apps.exited) + case collectorExit: + close(collector.exited) + case drainTimeout: + cutAllowed = true + timers.timer(testDrain) <- time.Now() + case workerdTimeout: + killAllowed = true + timers.timer(workerdGrace) <- time.Now() + } + check(snapshot()) + } + stopErr := await(t, stopped) + final := snapshot() + check(final) + if !slices.Contains(final, "reap collector") || stopErr == nil || + (!errors.Is(stopErr, product.err) && !slices.Contains(final, "kill product")) { + t.Fatalf("incorrect terminal outcome: trace %v, error %v", final, stopErr) + } + }) + } + permutations(0) +} From ae64459dfb8fa38dfd8fadc5000d1e80c9827ac1 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:20:56 -0800 Subject: [PATCH 2/8] End open-ended public streams when drain starts --- apps/hosted/self-host/README.md | 8 +++ apps/hosted/self-host/native/main.go | 3 +- apps/hosted/self-host/native/shutdown.go | 49 ++++++++++++++++--- apps/hosted/self-host/native/shutdown_test.go | 31 +++++++++++- 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index c325aa3858..ba04318b00 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -399,6 +399,14 @@ To stop the server while retaining its data: docker compose -f apps/hosted/self-host/compose.yaml down ``` +On SIGTERM or SIGINT the native host stops accepting new requests and gives +in-flight work up to 10 seconds to finish. Open GET event streams and NDJSON +streams end immediately, as do upgraded connections. A second signal ends the request +drain immediately without shortening the child processes' grace periods. The +product and apps get 10 seconds together to exit, then the telemetry collector +gets 3 seconds. If a child has to be killed, the host names it in the log and +exits nonzero. + ## App catalog Onboarding and **Add app** list Executor's official apps from the published diff --git a/apps/hosted/self-host/native/main.go b/apps/hosted/self-host/native/main.go index 77a596a41d..23eb2ebf5e 100644 --- a/apps/hosted/self-host/native/main.go +++ b/apps/hosted/self-host/native/main.go @@ -875,7 +875,8 @@ func serve(mode string) error { parts := shutdownParts{ signals: signals, serverError: serverError, stalled: stalled, endWatching: endWatching, endAnnouncing: stopAnnouncing, joinWatching: watchers.Wait, - public: public, listener: publicListener, edge: edge, + endStreams: func() { close(edge.ending) }, + public: public, listener: publicListener, edge: edge, closeIdle: proxy.Transport.(*http.Transport).CloseIdleConnections, closeBridge: closeBridge, product: product, productExited: product.exited, collector: telemetryCollector, } diff --git a/apps/hosted/self-host/native/shutdown.go b/apps/hosted/self-host/native/shutdown.go index f08dfb6cdf..d1605e9956 100644 --- a/apps/hosted/self-host/native/shutdown.go +++ b/apps/hosted/self-host/native/shutdown.go @@ -23,7 +23,10 @@ const ( cutGrace = time.Second ) -var errStopping = errors.New("Executor is stopping") +var ( + errStopping = errors.New("Executor is stopping") + errEndStream = errors.New("ending open-ended response") +) func stoppingResponse(w http.ResponseWriter) { w.Header().Set("Retry-After", "5") @@ -31,8 +34,9 @@ func stoppingResponse(w http.ResponseWriter) { } type edge struct { - base context.Context - cut context.CancelCauseFunc + base context.Context + cut context.CancelCauseFunc + ending chan struct{} mu sync.Mutex active map[*edgeRequest]struct{} @@ -44,13 +48,14 @@ type edgeRequestKey struct{} type edgeRequest struct { method string media string + cancel context.CancelCauseFunc } func newEdge() *edge { base, cut := context.WithCancelCause(context.Background()) idle := make(chan struct{}) close(idle) - return &edge{base: base, cut: cut, active: make(map[*edgeRequest]struct{}), idle: idle} + return &edge{base: base, cut: cut, ending: make(chan struct{}), active: make(map[*edgeRequest]struct{}), idle: idle} } func (e *edge) handler(next http.Handler) http.Handler { @@ -65,9 +70,12 @@ func (e *edge) handler(next http.Handler) http.Handler { if len(e.active) == 0 { e.idle = make(chan struct{}) } + ctx, cancel := context.WithCancelCause(r.Context()) + request.cancel = cancel e.active[request] = struct{}{} e.mu.Unlock() defer func() { + cancel(nil) e.mu.Lock() delete(e.active, request) if len(e.active) == 0 { @@ -75,24 +83,44 @@ func (e *edge) handler(next http.Handler) http.Handler { } e.mu.Unlock() }() - next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), edgeRequestKey{}, request))) + next.ServeHTTP(w, r.WithContext(context.WithValue(ctx, edgeRequestKey{}, request))) }) } func (e *edge) response(r *http.Response) error { media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) - if r.StatusCode == http.StatusSwitchingProtocols { + upgrade := r.StatusCode == http.StatusSwitchingProtocols + ctx := r.Request.Context() + openEnded := r.Request.Method == http.MethodGet && + (media == "text/event-stream" || media == "application/x-ndjson") + if upgrade { media = "upgrade" } else { if media == "" { media = "unspecified" } - r.Body = &cutBody{ReadCloser: r.Body, ctx: r.Request.Context()} + r.Body = &cutBody{ReadCloser: r.Body, ctx: ctx} } - if request, ok := r.Request.Context().Value(edgeRequestKey{}).(*edgeRequest); ok { + if request, ok := ctx.Value(edgeRequestKey{}).(*edgeRequest); ok { e.mu.Lock() request.media = media e.mu.Unlock() + if openEnded || upgrade { + if !upgrade { + // We may end before the advertised length, so frame it as a stream. + r.Header.Del("Content-Length") + r.ContentLength = -1 + } + go func() { + select { + case <-e.ending: + // Cancels a blocked transport read, or closes the backend of an upgrade + // through ReverseProxy, without losing its half-close support. + request.cancel(errEndStream) + case <-ctx.Done(): + } + }() + } } return nil } @@ -104,6 +132,9 @@ type cutBody struct { func (b *cutBody) Read(buf []byte) (int, error) { n, err := b.ReadCloser.Read(buf) + if errors.Is(context.Cause(b.ctx), errEndStream) { + return 0, io.EOF + } if err != nil && errors.Is(context.Cause(b.ctx), errStopping) { // ReverseProxy still aborts the response, but does not log an intentional cut // as an upstream read error or present the truncated body as complete. @@ -194,6 +225,7 @@ type shutdownParts struct { stalled <-chan error endWatching func() endAnnouncing func() + endStreams func() joinWatching func() public publicServer listener io.Closer @@ -239,6 +271,7 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { parts.apps.hold() } parts.collector.hold() + parts.endStreams() // Close explicitly as Serve may not yet have registered the listener. parts.listener.Close() ctx, cancel := context.WithCancel(context.Background()) diff --git a/apps/hosted/self-host/native/shutdown_test.go b/apps/hosted/self-host/native/shutdown_test.go index 74f8519748..cb5efafece 100644 --- a/apps/hosted/self-host/native/shutdown_test.go +++ b/apps/hosted/self-host/native/shutdown_test.go @@ -91,7 +91,11 @@ func runShutdownChild(role, name, journal, socket, edgeAddress string) { case <-r.Context().Done(): } default: - w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + media := "text/event-stream; charset=utf-8" + if r.URL.Path == "/ndjson" { + media = "application/x-ndjson" + } + w.Header().Set("Content-Type", media) fmt.Fprint(w, "data: open\n\n") w.(http.Flusher).Flush() <-r.Context().Done() @@ -236,6 +240,7 @@ func newShutdownFixture(t *testing.T) *shutdownFixture { f.parts = shutdownParts{ signals: f.signals, stalled: f.stalled, public: public, listener: listener, edge: edge, endWatching: func() { close(f.draining) }, endAnnouncing: func() {}, joinWatching: func() {}, + endStreams: func() { close(edge.ending) }, closeIdle: func() { proxy.Transport.(*http.Transport).CloseIdleConnections() if errors.Is(context.Cause(edge.base), errStopping) { @@ -503,6 +508,8 @@ func TestStopCutsAHijackedTunnelAtTheDeadline(t *testing.T) { // A POST keeps the drain open even though http.Server ignores the hijacked tunnel. f.request(http.MethodPost, "/stream") tunnel, _ := openTestTunnel(t, strings.TrimPrefix(f.url, "http://")) + // Exercise the BaseContext cut independently of the drain-start policy. + f.parts.endStreams = func() {} result := f.start() f.signals <- syscall.SIGTERM f.timers.timer(testDrain) <- time.Now() @@ -512,6 +519,27 @@ func TestStopCutsAHijackedTunnelAtTheDeadline(t *testing.T) { f.clean(result) } +func TestStopEndsOpenEndedStreamsAtOnce(t *testing.T) { + f := newShutdownFixture(t) + f.child("drains", "product") + f.child("quick", "collector") + eventStream := f.request(http.MethodGet, "/stream") + ndjson := f.request(http.MethodGet, "/ndjson") + tunnel, _ := openTestTunnel(t, strings.TrimPrefix(f.url, "http://")) + result := f.start() + f.signals <- syscall.SIGTERM + for _, response := range []*http.Response{eventStream, ndjson} { + body, err := io.ReadAll(response.Body) + if err != nil || len(body) == 0 { + t.Fatalf("stream did not end with a clean EOF: %q (%v)", body, err) + } + } + if _, err := tunnel.ReadByte(); !errors.Is(err, io.EOF) { + t.Fatalf("upgrade not closed at drain start: %v", err) + } + f.clean(result) +} + func TestSupervisorDoesNotRestartAfterHold(t *testing.T) { for _, inBackoff := range []bool{false, true} { t.Run(fmt.Sprintf("backoff=%t", inBackoff), func(t *testing.T) { @@ -750,6 +778,7 @@ func TestShutdownKeepsItsOrderForEveryEventOrder(t *testing.T) { }), endWatching: func() { log("watchdogs off") }, endAnnouncing: func() { log("announcer off") }, + endStreams: func() {}, joinWatching: func() {}, closeIdle: func() { if errors.Is(context.Cause(e.base), errStopping) { From df3c36146c8ca2d83b065e131664feed6b071354 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:20:56 -0800 Subject: [PATCH 3/8] Verify native shutdown in fork CI and a packaged runtime scenario --- .github/workflows/fork-ci.yml | 15 ++ e2e/test-plan.ts | 13 + e2e/tests/self-host-native-telemetry.spec.ts | 270 ++++++++++++++++++- 3 files changed, 297 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index 7ba640c447..cfa63ab317 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -16,6 +16,21 @@ permissions: contents: read jobs: + native: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: apps/hosted/self-host/native/go.mod + cache-dependency-path: apps/hosted/self-host/native/go.sum + - name: Vet and race-test the Go host + working-directory: apps/hosted/self-host/native + run: go vet ./... && go test -race -count=1 ./... + check: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 9dab3784dd..1466b740f8 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -203,6 +203,19 @@ export const scenarios = plan({ cloud: na("Cloud exports to Axiom, not a bundled collector."), }, }, + selfHostNativeGracefulShutdown: { + file: "self-host-native-telemetry.spec.ts", + title: "Native self-host exits cleanly on SIGTERM with open streams", + targets: { + "self-host": { + status: "not-run", + reason: + "Runs against the separately prepared Go/workerd runtime through e2e/self-host-native.config.ts; the ordinary self-host runner uses Bun.", + }, + local: na("Local does not use the Go/workerd host."), + cloud: na("Cloud does not own the Worker process lifecycle."), + }, + }, teamRegistry: { fixtures: "actors", file: "team-registry.spec.ts", diff --git a/e2e/tests/self-host-native-telemetry.spec.ts b/e2e/tests/self-host-native-telemetry.spec.ts index 30cdc3168f..497236f6a3 100644 --- a/e2e/tests/self-host-native-telemetry.spec.ts +++ b/e2e/tests/self-host-native-telemetry.spec.ts @@ -3,10 +3,12 @@ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { randomBytes } from "node:crypto"; import { request } from "node:http"; -import { Effect, Fiber, Schedule, Schema } from "effect"; +import { Effect, Fiber, FileSystem, Schedule, Schema } from "effect"; import { FetchHttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { appsManifest } from "../support/apps-release.ts"; import { collectorRequest, nativeSelfHost } from "../support/native-self-host.ts"; +import { lockedNpmRegistry } from "../support/npm-registry.ts"; import { driver } from "../support/platform.ts"; import { scenarios } from "../test-plan.ts"; @@ -252,3 +254,269 @@ it.live(scenarios.selfHostNativeTelemetryLifecycle.title, () => }), ).pipe(Effect.provide(NodeServices.layer), Effect.provide(FetchHttpClient.layer)), ); + +type NativeRequest = { + readonly path: string; + readonly headers?: Readonly>; + readonly body?: object; +}; + +/** Dial the fixture listener directly, including when the app has a different Host. */ +const nativeRequest = (origin: string, options: NativeRequest) => { + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + return request(new URL(options.path, origin), { + method: body === undefined ? "GET" : "POST", + headers: { + ...options.headers, + ...(body === undefined + ? {} + : { "content-type": "application/json", "content-length": Buffer.byteLength(body) }), + }, + }).end(body); +}; + +const nativeReply = (origin: string, options: NativeRequest) => + driver( + `${options.body === undefined ? "GET" : "POST"} ${new URL(options.path, origin).pathname}`, + () => + new Promise<{ response: import("node:http").IncomingMessage; text: string }>( + (resolve, reject) => { + nativeRequest(origin, options) + .once("error", reject) + .once("response", (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.once("error", reject); + response.once("end", () => + resolve({ response, text: Buffer.concat(chunks).toString("utf8") }), + ); + }); + }, + ), + ); + +const sessionCookies = (headers: import("node:http").IncomingHttpHeaders) => + (headers["set-cookie"] ?? []).map((cookie) => cookie.split(";")[0]).join("; "); + +const streamEvent = () => { + let resolve!: (value: A) => void; + const promise = new Promise((complete) => { + resolve = complete; + }); + return { promise, resolve }; +}; + +const liveStream = ( + origin: string, + options: NativeRequest, + firstFrame: (text: string) => boolean, +) => + Effect.gen(function* () { + const stream = yield* Effect.acquireRelease( + Effect.sync(() => { + const outgoing = nativeRequest(origin, options); + const ready = streamEvent(); + const finished = streamEvent(); + let ended = false; + outgoing.once("error", (error) => { + ready.resolve(String(error)); + ended = true; + finished.resolve(); + }); + outgoing.once("response", (response) => { + const media = response.headers["content-type"]?.split(";")[0]; + if (response.statusCode !== 200 || media !== "text/event-stream") { + ready.resolve(`expected event stream, got ${response.statusCode} ${media}`); + response.resume(); + } else { + let text = ""; + response.setEncoding("utf8"); + const received = (chunk: string) => { + text += chunk; + if (firstFrame(text)) { + ready.resolve("open"); + response.off("data", received); + response.resume(); + } + }; + response.on("data", received); + } + response.once("error", () => {}); + response.once("close", () => { + ended = true; + ready.resolve("stream closed before its first frame"); + finished.resolve(); + }); + }); + return { outgoing, ready: ready.promise, finished: finished.promise, ended: () => ended }; + }), + ({ outgoing }) => Effect.sync(() => outgoing.destroy()), + ); + const ready = yield* driver("wait for a stream frame", () => stream.ready).pipe( + Effect.timeout("10 seconds"), + ); + expect(ready, options.path).toBe("open"); + return stream; + }); + +it.live(scenarios.selfHostNativeGracefulShutdown.title, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const registry = yield* lockedNpmRegistry; + const host = yield* nativeSelfHost({ EXECUTOR_NPM_REGISTRY: `${registry.url}/` }); + const send = (options: NativeRequest) => nativeReply(host.origin, options); + const json = ( + schema: S, + reply: Effect.Success>, + ) => { + expect(reply.response.statusCode).toBe(200); + return Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(reply.text); + }; + const setup = yield* send({ + path: "/api/auth/self-host/setup", + headers: { origin: host.origin }, + body: { + name: "Shutdown Owner", + email: "shutdown@example.test", + password: "Synthetic-shutdown-password-123!", + organizationName: "Shutdown lab", + }, + }); + expect(setup.response.statusCode).toBe(200); + const owner = { origin: host.origin, cookie: sessionCookies(setup.response.headers) }; + const [organization] = yield* json( + Schema.NonEmptyArray(Schema.Struct({ id: Schema.String })), + yield* send({ path: "/api/auth/organization/list", headers: owner }), + ); + const key = yield* json( + Schema.Struct({ key: Schema.String }), + yield* send({ + path: "/api/auth/api-key/create", + headers: owner, + body: { name: "Shutdown MCP" }, + }), + ); + const apps = `/api/organizations/${organization.id}/apps`; + const app = yield* json( + Schema.Struct({ id: Schema.String }), + yield* send({ + path: `${apps}/deploy`, + headers: owner, + body: { + name: "Shutdown subscription", + files: [ + appsManifest, + { + path: "index.ts", + content: `import { defineApp, number, object, query, router } from "apps"; +export const latest = query({ input: object({}), output: object({ count: number() }) }, async ({ sql }) => + sql.exec("SELECT COUNT(*) AS count FROM notes").one() as { count: number }); +export default defineApp({}, { tools: router({ latest }) });`, + }, + { + path: "migrations/0001_notes.sql", + content: "CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL);\n", + }, + { + path: "ui/index.html", + content: + "Shutdown", + }, + ], + }, + }), + ); + const appHost = yield* send({ path: `${apps}/${app.id}/ui`, headers: owner }).pipe( + Effect.flatMap((reply) => + json(Schema.Struct({ status: Schema.String, url: Schema.NullOr(Schema.String) }), reply), + ), + Effect.flatMap(({ status, url }) => + status === "ready" && url !== null + ? Effect.succeed(new URL(url).host) + : Effect.fail("The app address is not ready"), + ), + Effect.retry({ schedule: Schedule.spaced("250 millis"), times: 40 }), + ); + const navigation = { accept: "text/html", "sec-fetch-mode": "navigate" }; + const redirectPath = (location: string | undefined) => { + const url = new URL(location ?? "", host.origin); + return `${url.pathname}${url.search}`; + }; + const visit = yield* send({ path: "/", headers: { ...navigation, host: appHost } }); + expect(visit.response.statusCode).toBe(302); + const approve = yield* send({ + path: redirectPath(visit.response.headers.location), + headers: { ...navigation, cookie: owner.cookie }, + }); + expect(approve.response.statusCode).toBe(302); + const signedIn = yield* send({ + path: redirectPath(approve.response.headers.location), + headers: { ...navigation, host: appHost, cookie: sessionCookies(visit.response.headers) }, + }); + expect(signedIn.response.statusCode).toBe(302); + const appSession = { host: appHost, cookie: sessionCookies(signedIn.response.headers) }; + const page = yield* send({ path: "/", headers: { ...navigation, ...appSession } }); + expect(page.response.statusCode).toBe(200); + const context = /]*id="executor-context"[^>]*>([^<]+)<\/script>/u.exec(page.text); + const { version } = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ version: Schema.String })), + )(context?.[1]); + + const subscription = yield* liveStream( + host.origin, + { + path: "/_executor/api/subscribe", + headers: { ...appSession, origin: `http://${appHost}`, accept: "text/event-stream" }, + body: { version, name: "latest", input: {} }, + }, + (text) => text.includes('"type":"snapshot"'), + ); + const events = yield* liveStream( + host.origin, + { + path: "/mcp", + // Streamable HTTP here exposes subscriptions/listen as POST, not GET /mcp. + headers: { + authorization: `Bearer ${key.key}`, + "x-executor-organization": organization.id, + "mcp-protocol-version": "2026-07-28", + "mcp-method": "subscriptions/listen", + accept: "application/json, text/event-stream", + }, + body: { + jsonrpc: "2.0", + id: 1, + method: "subscriptions/listen", + params: { + _meta: { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": {}, + }, + notifications: { toolsListChanged: true }, + }, + }, + }, + (text) => text.startsWith(":") || text.includes("data:"), + ); + expect(events.ended(), "MCP is still streaming").toBe(false); + expect(subscription.ended(), "the app is still subscribed").toBe(false); + + const signalled = performance.now(); + yield* Effect.sync(() => process.kill(host.pid, "SIGTERM")); + // Observe the whole stop even on a regression, but fail the user-facing 5s bound below. + const exitCode = yield* host.exitCode.pipe(Effect.timeout("25 seconds")); + const elapsed = Math.round(performance.now() - signalled); + yield* driver("both streams ended", () => + Promise.all([events.finished, subscription.finished]), + ).pipe(Effect.timeout("1 second")); + const log = yield* fs.readFileString(`${host.dataDirectory}/../server.log`); + yield* Effect.log( + `SIGTERM to host exit: ${elapsed}ms, exit code ${exitCode}, log at ${host.dataDirectory}/../server.log`, + ); + expect.soft(elapsed, "SIGTERM to host exit").toBeLessThan(5_000); + expect.soft(exitCode).toBe(0); + expect.soft(log).not.toMatch(/killed|ReverseProxy read error/u); + }), + ).pipe(Effect.provide(NodeServices.layer), Effect.provide(FetchHttpClient.layer)), +); From f520584db63cd7c50021d05bbaa9d09fe611a6e9 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:22:53 -0800 Subject: [PATCH 4/8] Assert the full POST drain in the shutdown scenario --- e2e/tests/self-host-native-telemetry.spec.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/e2e/tests/self-host-native-telemetry.spec.ts b/e2e/tests/self-host-native-telemetry.spec.ts index 497236f6a3..ee4586aa15 100644 --- a/e2e/tests/self-host-native-telemetry.spec.ts +++ b/e2e/tests/self-host-native-telemetry.spec.ts @@ -504,17 +504,22 @@ export default defineApp({}, { tools: router({ latest }) });`, const signalled = performance.now(); yield* Effect.sync(() => process.kill(host.pid, "SIGTERM")); - // Observe the whole stop even on a regression, but fail the user-facing 5s bound below. + // Observe the whole stop even if it exceeds the expected shutdown window. const exitCode = yield* host.exitCode.pipe(Effect.timeout("25 seconds")); - const elapsed = Math.round(performance.now() - signalled); + const elapsed = performance.now() - signalled; yield* driver("both streams ended", () => Promise.all([events.finished, subscription.finished]), ).pipe(Effect.timeout("1 second")); const log = yield* fs.readFileString(`${host.dataDirectory}/../server.log`); yield* Effect.log( - `SIGTERM to host exit: ${elapsed}ms, exit code ${exitCode}, log at ${host.dataDirectory}/../server.log`, + `SIGTERM to host exit: ${Math.round(elapsed)}ms, exit code ${exitCode}, log at ${host.dataDirectory}/../server.log`, ); - expect.soft(elapsed, "SIGTERM to host exit").toBeLessThan(5_000); + // drainGrace in apps/hosted/self-host/native/shutdown.go gives POST streams 10 seconds. + const drainGraceMs = 10_000; + expect + .soft(elapsed, "POST streams get the drain window") + .toBeGreaterThanOrEqual(drainGraceMs - 1_000); + expect.soft(elapsed, "SIGTERM to host exit").toBeLessThanOrEqual(drainGraceMs + 5_000); expect.soft(exitCode).toBe(0); expect.soft(log).not.toMatch(/killed|ReverseProxy read error/u); }), From df9f70ffa14b841dc572424643cf7712a606903f Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:35:53 -0800 Subject: [PATCH 5/8] Address native shutdown review findings --- apps/hosted/self-host/README.md | 20 +- apps/hosted/self-host/native/main.go | 97 ++--- apps/hosted/self-host/native/main_test.go | 32 +- apps/hosted/self-host/native/shutdown.go | 28 +- apps/hosted/self-host/native/shutdown_test.go | 353 ++++++++++++------ 5 files changed, 311 insertions(+), 219 deletions(-) diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index ba04318b00..472c6a32b8 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -42,12 +42,15 @@ session and encryption keys in the persistent volume on first boot. ```sh docker pull ghcr.io/usefulsoftwareco/executor-selfhost:beta -docker run --detach --name executor-v2 --init --restart unless-stopped \ +docker run --detach --name executor-v2 --init --restart unless-stopped --stop-timeout 30 \ --publish 127.0.0.1:4400:4400 \ --volume executor-v2-data:/app/data \ ghcr.io/usefulsoftwareco/executor-selfhost:beta ``` +Docker's default 10-second stop timeout is shorter than the host's request drain +plus child process grace periods; keep `--stop-timeout 30` when recreating the container. + Open `http://localhost:4400` and create the first administrator. The container includes the dashboard, API, MCP server, database, and app runtime. For a pinned release, use a published version such as `2.0.0-beta.N` instead of the moving @@ -401,8 +404,9 @@ docker compose -f apps/hosted/self-host/compose.yaml down On SIGTERM or SIGINT the native host stops accepting new requests and gives in-flight work up to 10 seconds to finish. Open GET event streams and NDJSON -streams end immediately, as do upgraded connections. A second signal ends the request -drain immediately without shortening the child processes' grace periods. The +responses end immediately with a clean EOF, even if a finite NDJSON response +has more records; upgraded connections also close immediately. A second signal +ends the request drain without shortening the child processes' grace periods. The product and apps get 10 seconds together to exit, then the telemetry collector gets 3 seconds. If a child has to be killed, the host names it in the log and exits nonzero. @@ -585,10 +589,12 @@ the body reaches a sleep before each wake. The product process has its own watchdog. Once the product has answered its first health check, the host probes `/health` every `EXECUTOR_WATCHDOG_INTERVAL` seconds (default 10). After -`EXECUTOR_WATCHDOG_FAILURES` failed probes in a row (default 9, about 90 seconds), it kills workerd -and exits non-zero. Run the container with a restart policy such as `restart: unless-stopped` (the -bundled compose file sets it), so a fresh process starts. `EXECUTOR_WATCHDOG=off` turns both -watchdogs off. +`EXECUTOR_WATCHDOG_FAILURES` failed probes in a row (default 9, about 90 seconds), it logs +`Executor watchdog: the product did not answer …` and cuts public requests without draining. +It then sends SIGTERM to workerd, gives the product and apps five seconds together to stop, +kills any that are still running, and exits non-zero. Run the container with a restart policy such +as `restart: unless-stopped` (the bundled compose file sets it), so a fresh process starts. +`EXECUTOR_WATCHDOG=off` turns both watchdogs off. `EXECUTOR_APP_PROCESS=shared` runs app code in the product's process as earlier releases did; then only the product watchdog applies, and a looping app restarts the whole host. Exports always use diff --git a/apps/hosted/self-host/native/main.go b/apps/hosted/self-host/native/main.go index 23eb2ebf5e..34f43f4873 100644 --- a/apps/hosted/self-host/native/main.go +++ b/apps/hosted/self-host/native/main.go @@ -761,11 +761,23 @@ func serve(mode string) error { apps.Env = append(apps.Env, handleSecret) return apps }, time.Second, 30*time.Second, time.After) - defer stopChildren(workerdGrace, time.After, apps) } else { command.Env = append(command.Env, handleSecret) } + stopStartingWorkerd := func(product *process) { + if apps != nil { + apps.hold() + } + if product != nil { + product.cmd.Process.Kill() + <-product.exited + } + if apps != nil { + stopChildren(workerdGrace, time.After, apps) + } + } if err = command.Start(); err != nil { + stopStartingWorkerd(nil) return err } product := waitProcess("Product process", command) @@ -786,8 +798,7 @@ func serve(mode string) error { } forget, err := recordTelemetrySocket(directory, collector) if err != nil { - command.Process.Kill() - <-product.exited + stopStartingWorkerd(product) return err } defer forget() @@ -808,23 +819,20 @@ func serve(mode string) error { publicListener, err := net.Listen("tcp", net.JoinHostPort(setting("HOST", "0.0.0.0"), port)) if err != nil { - command.Process.Kill() - <-product.exited - if apps != nil { - stopChildren(workerdGrace, time.After, apps) - } + telemetryCollector.hold() + stopStartingWorkerd(product) stopChildren(collectorGrace, time.After, telemetryCollector) return err } socketPath := filepath.Join(temporary, "product.sock") edge := newEdge() - addressOf := func(request *http.Request) string { - return clientAddress(request.RemoteAddr, request.Header.Values("X-Forwarded-For"), proxies) - } + var proxy *httputil.ReverseProxy if trustedProxy != nil { - addressOf = trustedProxy.clientIP + proxy = productProxyFor(socketPath, workerdIdleTimeout, trustedProxy.clientIP) + } else { + proxy = productProxy(socketPath, workerdIdleTimeout, proxies...) } - proxy := productProxyFor(socketPath, workerdIdleTimeout, addressOf, edge) + proxy.ModifyResponse = edge.response opened := make(chan struct{}) public := &http.Server{ Handler: edge.handler(startingGate(opened, publicHandler(proxy))), @@ -835,8 +843,8 @@ func serve(mode string) error { // One app isolate in a synchronous loop holds workerd's only JavaScript thread, so the product // and every other app stop answering and no timeout inside workerd can fire. OSS workerd does // not enforce CPU limits, so the host bounds the outage instead: once the product has answered, - // a run of failed health probes kills workerd, this host exits non-zero, and the container's - // restart policy starts a fresh process. + // a run of failed health probes starts a bounded shutdown. This host exits non-zero and + // the container's restart policy starts a fresh process. watchCtx, endWatching := context.WithCancel(context.Background()) var watchers sync.WaitGroup stalled := make(chan error, 1) @@ -844,7 +852,7 @@ func serve(mode string) error { probe := healthProbe(watchCtx, socketPath, min(interval, 5*time.Second), false) watchers.Go(func() { watchdog(probe, interval, failures, time.After, watchCtx.Done(), func() { - stalled <- fmt.Errorf("watchdog: the product did not answer %d health checks in a row (every %s); workerd is unresponsive", failures, interval) + reportProductStall(os.Stderr, stalled, failures, interval) }) }) } @@ -870,7 +878,7 @@ func serve(mode string) error { } announceCtx, stopAnnouncing := context.WithCancel(context.Background()) watchers.Go(func() { - announceReadyContext(announceCtx, socketPath, values["BETTER_AUTH_URL"], startupProgressInterval, time.Second, os.Stderr, opened) + announceReady(announceCtx, socketPath, values["BETTER_AUTH_URL"], startupProgressInterval, time.Second, os.Stderr, opened) }) parts := shutdownParts{ signals: signals, serverError: serverError, stalled: stalled, @@ -898,21 +906,8 @@ const startupProgressInterval = 15 * time.Second // would otherwise initialize only once the first visitor or health probe arrived. It asks the // product's /health until one answers 200, waiting retry after the first failure and doubling up // to 30 seconds, and logs every progress interval while it waits. It closes opened when the -// product first answers, ready or not, and returns once the product is ready or done closes. -func announceReady(socket, origin string, progress, retry time.Duration, log io.Writer, opened chan<- struct{}, done <-chan struct{}) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { - select { - case <-done: - cancel() - case <-ctx.Done(): - } - }() - announceReadyContext(ctx, socket, origin, progress, retry, log, opened) -} - -func announceReadyContext(ctx context.Context, socket, origin string, progress, retry time.Duration, log io.Writer, opened chan<- struct{}) { +// product first answers, ready or not, and returns once the product is ready or ctx is cancelled. +func announceReady(ctx context.Context, socket, origin string, progress, retry time.Duration, log io.Writer, opened chan<- struct{}) { started := time.Now() fmt.Fprintln(log, "Executor is starting. The first start initializes the database, which can take a minute or more on slower hardware.") // A fresh connection each time: a pooled one could reach workerd as it closes it. @@ -1084,11 +1079,11 @@ func (c *workerdConn) Close() error { func productProxy(socket string, upstreamIdleTimeout time.Duration, trusted ...netip.Prefix) *httputil.ReverseProxy { return productProxyFor(socket, upstreamIdleTimeout, func(request *http.Request) string { return clientAddress(request.RemoteAddr, request.Header.Values("X-Forwarded-For"), trusted) - }, nil) + }) } -func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf func(*http.Request) string, edge *edge) *httputil.ReverseProxy { - proxy := &httputil.ReverseProxy{ +func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf func(*http.Request) string) *httputil.ReverseProxy { + return &httputil.ReverseProxy{ Rewrite: func(request *httputil.ProxyRequest) { request.SetURL(&url.URL{Scheme: "http", Host: "product.internal"}) request.Out.Host = request.In.Host @@ -1127,10 +1122,6 @@ func productProxyFor(socket string, upstreamIdleTimeout time.Duration, addressOf http.Error(w, "Executor did not complete the request", http.StatusBadGateway) }, } - if edge != nil { - proxy.ModifyResponse = edge.response - } - return proxy } // watchdogSettings reads EXECUTOR_WATCHDOG_INTERVAL (seconds, default 10) and @@ -1167,13 +1158,8 @@ func appWatchdogSettings() (time.Duration, int, bool) { return time.Duration(seconds * float64(time.Second)), failures, true } -// productProbe asks the product's own socket for /health, bypassing the public proxy, and -// reports whether it answered 200 within timeout. -func productProbe(socket string, timeout time.Duration) func() bool { - return healthProbe(context.Background(), socket, timeout, false) -} - -// Any response from apps shows that its JavaScript thread is free, even a refusal. +// healthProbe bypasses the public proxy and asks the child's socket for /health. +// For the apps process any response shows that its JavaScript thread is free, even a refusal. func healthProbe(ctx context.Context, socket string, timeout time.Duration, anyAnswer bool) func() bool { client := http.Client{ Timeout: timeout, @@ -1230,20 +1216,11 @@ func watchdog(probe func() bool, interval time.Duration, failures int, after fun } } -// supervise keeps the telemetry collector running beside the product; see superviseProcess. -func supervise(newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) func() { - stop, _ := superviseProcess("Telemetry collector", newCommand, minDelay, maxDelay, after) - return stop -} - -// superviseProcess keeps an auxiliary process running beside the product: it restarts the process -// after an exit, waiting from minDelay up to maxDelay between attempts, and the product never -// waits for it. A process that ran longer than maxDelay restarts after minDelay again. after -// starts each wait. It returns a function that stops the process and waits for it to exit, and one -// that kills the running process so it restarts. -func superviseProcess(name string, newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) (func(), func()) { - s := startSupervisor(name, newCommand, minDelay, maxDelay, after) - return func() { stopChildren(workerdGrace, time.After, s) }, s.restart +// Keep the detection log separate from the final shutdown result: an operator needs to +// see why the host is restarting before it waits on any child. +func reportProductStall(log io.Writer, stalled chan<- error, failures int, interval time.Duration) { + fmt.Fprintf(log, "Executor watchdog: the product did not answer %d health checks in a row (every %s); workerd is unresponsive, restarting it\n", failures, interval) + stalled <- errors.New("product watchdog detected unresponsive workerd") } // healthTimeout is how long `executor-host health` waits for /health. Every response, /health diff --git a/apps/hosted/self-host/native/main_test.go b/apps/hosted/self-host/native/main_test.go index 8c2d9d5302..8c9adbe288 100644 --- a/apps/hosted/self-host/native/main_test.go +++ b/apps/hosted/self-host/native/main_test.go @@ -2,6 +2,7 @@ package main import ( "bufio" + "context" "fmt" "io" "net" @@ -396,7 +397,7 @@ func TestSuperviseRestartsAnExitedProcessWithBackoff(t *testing.T) { return time.After(0) } var starts atomic.Int32 - stop := supervise(func() *exec.Cmd { + collector := startSupervisor("Telemetry collector", func() *exec.Cmd { // The fifth process outlives the maximum wait; every other one exits at once. if starts.Add(1) == 5 { return exec.Command("sh", "-c", "sleep 0.6; exit 1") @@ -407,7 +408,7 @@ func TestSuperviseRestartsAnExitedProcessWithBackoff(t *testing.T) { for starts.Load() < 7 && time.Now().Before(deadline) { time.Sleep(10 * time.Millisecond) } - stop() + stopChildren(workerdGrace, time.After, collector) mutex.Lock() recorded := append([]time.Duration(nil), waits...) mutex.Unlock() @@ -429,12 +430,12 @@ func TestSuperviseRestartsAnExitedProcessWithBackoff(t *testing.T) { func TestSuperviseStopsARunningProcess(t *testing.T) { marker := filepath.Join(t.TempDir(), "stopped") - stop := supervise(func() *exec.Cmd { + collector := startSupervisor("Telemetry collector", func() *exec.Cmd { return exec.Command("sh", "-c", `trap 'touch "$0"; exit 0' TERM; while :; do sleep 0.05; done`, marker) }, time.Second, time.Second, time.After) time.Sleep(200 * time.Millisecond) returned := make(chan struct{}) - go func() { stop(); close(returned) }() + go func() { stopChildren(workerdGrace, time.After, collector); close(returned) }() select { case <-returned: case <-time.After(5 * time.Second): @@ -519,7 +520,7 @@ func TestAnnounceReadyOpensTheProductAtBootAndSaysWhenItIsReady(t *testing.T) { opened := make(chan struct{}) returned := make(chan struct{}) go func() { - announceReady(socket, "https://executor.example.test:4443", 30*time.Millisecond, 5*time.Millisecond, &log, opened, make(chan struct{})) + announceReady(context.Background(), socket, "https://executor.example.test:4443", 30*time.Millisecond, 5*time.Millisecond, &log, opened) close(returned) }() select { @@ -561,9 +562,9 @@ func TestAnnounceReadyOpensTheGateOnTheFirstAnswerEvenWhenItFails(t *testing.T) })) opened := make(chan struct{}) - done := make(chan struct{}) - defer close(done) - go announceReady(socket, "http://localhost:4400", time.Hour, time.Hour, io.Discard, opened, done) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go announceReady(ctx, socket, "http://localhost:4400", time.Hour, time.Hour, io.Discard, opened) select { case <-opened: case <-time.After(5 * time.Second): @@ -629,14 +630,15 @@ func TestAnnounceReadyStopsWithTheHost(t *testing.T) { var log syncBuffer opened := make(chan struct{}) - done := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() returned := make(chan struct{}) go func() { - announceReady(socket, "http://localhost:4400", time.Hour, time.Millisecond, &log, opened, done) + announceReady(ctx, socket, "http://localhost:4400", time.Hour, time.Millisecond, &log, opened) close(returned) }() <-requested - close(done) + cancel() select { case <-returned: case <-time.After(time.Second): @@ -836,7 +838,7 @@ func TestProductProbeReportsOnlyOK(t *testing.T) { })} go server.Serve(listener) defer server.Close() - probe := productProbe(socket, time.Second) + probe := healthProbe(context.Background(), socket, time.Second, false) if !probe() { t.Fatal("a 200 from /health is an answer") } @@ -852,12 +854,12 @@ func TestProductProbeReportsOnlyOK(t *testing.T) { func TestSuperviseProcessKillRestartsOnlyThatProcess(t *testing.T) { var starts atomic.Int32 - stop, kill := superviseProcess("Test process", func() *exec.Cmd { + child := startSupervisor("Test process", func() *exec.Cmd { starts.Add(1) // A process that never exits on its own, like a workerd whose thread is stuck. return exec.Command("sh", "-c", "while :; do sleep 0.05; done") }, 10*time.Millisecond, 50*time.Millisecond, func(time.Duration) <-chan time.Time { return time.After(0) }) - defer stop() + defer stopChildren(workerdGrace, time.After, child) waitFor := func(count int32) { deadline := time.Now().Add(5 * time.Second) for starts.Load() < count && time.Now().Before(deadline) { @@ -870,7 +872,7 @@ func TestSuperviseProcessKillRestartsOnlyThatProcess(t *testing.T) { waitFor(1) // Let the first process be registered as running before killing it. time.Sleep(100 * time.Millisecond) - kill() + child.restartUnlessStopping(context.Background(), func() {}) waitFor(2) time.Sleep(100 * time.Millisecond) if count := starts.Load(); count != 2 { diff --git a/apps/hosted/self-host/native/shutdown.go b/apps/hosted/self-host/native/shutdown.go index d1605e9956..60bd17ccf8 100644 --- a/apps/hosted/self-host/native/shutdown.go +++ b/apps/hosted/self-host/native/shutdown.go @@ -17,6 +17,7 @@ import ( const ( drainGrace = 10 * time.Second workerdGrace = 10 * time.Second + stallGrace = 5 * time.Second collectorGrace = 3 * time.Second // Let cancelled handlers release their upstreams and flush a 503 before Close tears // down their downstreams. A client that isn't reading cannot extend this wait. @@ -133,9 +134,9 @@ type cutBody struct { func (b *cutBody) Read(buf []byte) (int, error) { n, err := b.ReadCloser.Read(buf) if errors.Is(context.Cause(b.ctx), errEndStream) { - return 0, io.EOF + return n, io.EOF } - if err != nil && errors.Is(context.Cause(b.ctx), errStopping) { + if err != nil && !errors.Is(err, io.EOF) && errors.Is(context.Cause(b.ctx), errStopping) { // ReverseProxy still aborts the response, but does not log an intentional cut // as an upstream read error or present the truncated body as complete. err = context.Canceled @@ -255,6 +256,7 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { // Serving. Only this select can act on a product watchdog report. var reason error skipDrain := false + grace := plan.workerd select { case <-parts.signals: case reason = <-parts.serverError: @@ -262,6 +264,7 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { skipDrain = true case reason = <-parts.stalled: skipDrain = true + grace = stallGrace } // Draining. Stop creating upstream work before waiting on existing requests. @@ -271,7 +274,6 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { parts.apps.hold() } parts.collector.hold() - parts.endStreams() // Close explicitly as Serve may not yet have registered the listener. parts.listener.Close() ctx, cancel := context.WithCancel(context.Background()) @@ -281,6 +283,7 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { close(drained) }() if !skipDrain { + parts.endStreams() select { case <-drained: case <-plan.after(plan.drain): @@ -307,7 +310,7 @@ func shutdown(parts shutdownParts, plan shutdownPlan) error { // Stopping workerd. Both processes share one deadline; neither waits on the // other to receive SIGTERM. Keep the bridge and collector alive for both. - err := errors.Join(reason, stopChildren(plan.workerd, plan.after, parts.product, parts.apps)) + err := errors.Join(reason, stopChildren(grace, plan.after, parts.product, parts.apps)) parts.closeBridge() parts.joinWatching() @@ -346,8 +349,6 @@ const ( running childState = iota held stopping - exited - killed ) type supervisor struct { @@ -361,6 +362,8 @@ type supervisor struct { stopErr error } +// startSupervisor restarts an auxiliary process after an exit, backing off from minDelay +// up to maxDelay. A process that ran longer than maxDelay resets the wait to minDelay. func startSupervisor(name string, newCommand func() *exec.Cmd, minDelay, maxDelay time.Duration, after func(time.Duration) <-chan time.Time) *supervisor { s := &supervisor{name: name, held: make(chan struct{}), finished: make(chan struct{})} go s.run(newCommand, minDelay, maxDelay, after) @@ -426,23 +429,14 @@ func (s *supervisor) stop(grace time.Duration, deadline <-chan struct{}) error { s.state = stopping p := s.current s.mu.Unlock() - wasKilled := stopProcess(p, deadline) - <-s.finished - s.mu.Lock() - defer s.mu.Unlock() - s.state = exited - if wasKilled { - s.state = killed + if stopProcess(p, deadline) { s.stopErr = killedChild(s.name, grace) } + <-s.finished }) return s.stopErr } -func (s *supervisor) restart() { - s.restartUnlessStopping(context.Background(), func() {}) -} - func (s *supervisor) restartUnlessStopping(ctx context.Context, report func()) { s.mu.Lock() defer s.mu.Unlock() diff --git a/apps/hosted/self-host/native/shutdown_test.go b/apps/hosted/self-host/native/shutdown_test.go index cb5efafece..5cf588fd84 100644 --- a/apps/hosted/self-host/native/shutdown_test.go +++ b/apps/hosted/self-host/native/shutdown_test.go @@ -19,6 +19,7 @@ import ( "sync/atomic" "syscall" "testing" + "testing/synctest" "time" ) @@ -53,7 +54,7 @@ func runShutdownChild(role, name, journal, socket, edgeAddress string) { journalEvent(journal, name+":"+event) fmt.Println(event) } - if role == "drains" { + if role == "drains" || role == "stalled" { var err error listener, err = net.Listen("unix", socket) if err != nil { @@ -115,7 +116,7 @@ func runShutdownChild(role, name, journal, socket, edgeAddress string) { } } notify("term") - if role == "stubborn" { + if role == "stubborn" || role == "stalled" { for range signals { } } @@ -223,7 +224,8 @@ func newShutdownFixture(t *testing.T) *shutdownFixture { t.Fatal(err) } edge := newEdge() - proxy := productProxyFor(f.socket, workerdIdleTimeout, func(*http.Request) string { return "127.0.0.1" }, edge) + proxy := productProxy(f.socket, workerdIdleTimeout) + proxy.ModifyResponse = edge.response public := &http.Server{ Handler: edge.handler(publicHandler(proxy)), BaseContext: func(net.Listener) context.Context { return edge.base }, @@ -565,8 +567,10 @@ func TestSupervisorDoesNotRestartAfterHold(t *testing.T) { if inBackoff { child.cmd.Process.Signal(syscall.SIGTERM) timer := await(t, backoff) + // Rescue a broken supervisor at cleanup only. The wait below must end + // without firing the backoff timer. + t.Cleanup(func() { timer <- time.Now() }) s.hold() - timer <- time.Now() } else { s.hold() reported := false @@ -623,6 +627,7 @@ func TestStopKillsOnlyAChildThatIgnoresSIGTERM(t *testing.T) { func TestStopAfterTheProductExitedSkipsTheDrain(t *testing.T) { f := newShutdownFixture(t) + ending := f.parts.edge.ending // A blocked handler makes waiting for graceful HTTP Shutdown observable. enter, leave := make(chan struct{}), make(chan struct{}) existing := f.parts.edge.handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -650,6 +655,11 @@ func TestStopAfterTheProductExitedSkipsTheDrain(t *testing.T) { if err := await(t, result); !errors.As(err, &exited) || exited.ExitCode() != 42 { t.Fatalf("want product exit 42, got %v", err) } + select { + case <-ending: + t.Fatal("product exit must abort responses, not end streams cleanly") + default: + } } func TestStopFindsChildrenAlreadyGone(t *testing.T) { @@ -683,6 +693,67 @@ func TestAStalledProductAfterTheStopBeganIsIgnored(t *testing.T) { } } +func TestStopAfterWatchdogStallCutsStreamsAndUsesShortGrace(t *testing.T) { + f := newShutdownFixture(t) + product := f.child("stalled", "product") + f.child("quick", "collector") + stream := f.request(http.MethodGet, "/stream") + result := f.start() + var log bytes.Buffer + reportProductStall(&log, f.stalled, 3, 2*time.Second) + const want = "Executor watchdog: the product did not answer 3 health checks in a row (every 2s); workerd is unresponsive, restarting it\n" + if log.String() != want { + t.Fatalf("watchdog detection log = %q; want %q", log.String(), want) + } + if _, err := io.ReadAll(stream.Body); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("stalled product stream must be aborted, got %v", err) + } + select { + case <-f.parts.edge.ending: + t.Fatal("watchdog must abort responses, not end streams cleanly") + default: + } + for event := await(t, product.events); event != "term"; event = await(t, product.events) { + if event != "request-ended" { + t.Fatalf("got %q waiting for product SIGTERM", event) + } + } + f.timers.timer(5 * time.Second) <- time.Now() + if err := await(t, result); err == nil || !strings.Contains(err.Error(), "product did not stop within 5s; killed") { + t.Fatalf("want watchdog failure with five-second escalation, got %v", err) + } +} + +type readerFunc func([]byte) (int, error) + +func (f readerFunc) Read(buf []byte) (int, error) { return f(buf) } + +func TestCutBodyPreservesCleanEOF(t *testing.T) { + ctx, cut := context.WithCancelCause(context.Background()) + body := cutBody{ctx: ctx, ReadCloser: io.NopCloser(readerFunc(func(buf []byte) (int, error) { + n := copy(buf, "last chunk") + cut(errStopping) + return n, io.EOF + }))} + got, err := io.ReadAll(&body) + if string(got) != "last chunk" || err != nil { + t.Fatalf("a response that completed at the cut returned %q, %v", got, err) + } +} + +func TestCutBodyDeliversTheLastReadWhenEndingAnOpenStream(t *testing.T) { + ctx, cut := context.WithCancelCause(context.Background()) + body := cutBody{ctx: ctx, ReadCloser: io.NopCloser(readerFunc(func(buf []byte) (int, error) { + n := copy(buf, "last chunk") + cut(errEndStream) + return n, nil + }))} + got, err := io.ReadAll(&body) + if string(got) != "last chunk" || err != nil { + t.Fatalf("an open-ended stream returned %q, %v; want last chunk and clean EOF", got, err) + } +} + type closeFunc func() func (f closeFunc) Close() error { f(); return nil } @@ -696,7 +767,6 @@ type modelChild struct { log func(string) exited chan struct{} err error - term chan struct{} } func (c *modelChild) hold() { c.log("hold " + c.name) } @@ -709,7 +779,6 @@ func (c *modelChild) stop(grace time.Duration, deadline <-chan struct{}) error { default: } c.log("term " + c.name) - close(c.term) select { case <-c.exited: return c.err @@ -720,132 +789,176 @@ func (c *modelChild) stop(grace time.Duration, deadline <-chan struct{}) error { default: } c.log("kill " + c.name) + <-c.exited // A kill is not a reap; the driver must acknowledge the exit. return killedChild(c.name, grace) } } func TestShutdownKeepsItsOrderForEveryEventOrder(t *testing.T) { - const ( - secondSignal = iota - productExit - appsExit - collectorExit - drainTimeout - workerdTimeout - ) - events := []int{secondSignal, productExit, appsExit, collectorExit, drainTimeout, workerdTimeout} - var permutations func(int) - permutations = func(from int) { - if from < len(events) { - for i := from; i < len(events); i++ { - events[from], events[i] = events[i], events[from] - permutations(from + 1) - events[from], events[i] = events[i], events[from] + // Drain has one winning event. Workerd exit events race a shared deadline, and the + // collector then has its own deadline. Enumerate those orders within their phases, + // not permutations that inject every child exit before reaching the stop phase. + workerdOrders := [][3]string{ + {"product", "apps", "timer"}, {"apps", "product", "timer"}, + {"product", "timer", "apps"}, {"apps", "timer", "product"}, + {"timer", "product", "apps"}, {"timer", "apps", "product"}, + } + for _, drain := range []string{"second-signal", "product-exit", "drain-timer"} { + for _, order := range workerdOrders { + for _, collectorTimesOut := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/collector-timeout=%t", drain, strings.Join(order[:], "-"), collectorTimesOut), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + checkShutdownOrder(t, drain, order, collectorTimesOut) + }) + }) } - return } - t.Run(fmt.Sprint(events), func(t *testing.T) { - var mu sync.Mutex - var trace []string - log := func(event string) { - mu.Lock() - defer mu.Unlock() - trace = append(trace, event) + } +} + +func checkShutdownOrder(t *testing.T, drain string, order [3]string, collectorTimesOut bool) { + var mu sync.Mutex + var trace []string + log := func(event string) { + mu.Lock() + defer mu.Unlock() + trace = append(trace, event) + } + child := func(name string) *modelChild { + return &modelChild{name: name, log: log, exited: make(chan struct{})} + } + product, apps, collector := child("product"), child("apps"), child("collector") + if drain == "product-exit" { + product.err = errors.New("product failed") + } + signals := make(chan os.Signal, 2) + timers := manualTimers{timers: make(map[time.Duration]chan time.Time)} + e := newEdge() + stopped := make(chan error, 1) + go func() { + stopped <- shutdown(shutdownParts{ + signals: signals, edge: e, public: modelPublic{func() { log("close public") }}, + product: product, productExited: product.exited, apps: apps, collector: collector, + listener: closeFunc(func() { log("close listener") }), + endWatching: func() { log("watchdogs off") }, + endAnnouncing: func() { log("announcer off") }, + endStreams: func() {}, + joinWatching: func() {}, + closeIdle: func() { + if errors.Is(context.Cause(e.base), errStopping) { + log("cut") + } + }, + closeBridge: func() { log("close bridge") }, + }, shutdownPlan{testDrain, workerdGrace, collectorGrace, timers.after, io.Discard}) + }() + + // Check exact effects after each event, once shutdown and all children have + // settled. No next event can retroactively permit an early cut, TERM or KILL. + expected := map[string]bool{} + check := func() { + t.Helper() + synctest.Wait() + has := func(event string) bool { return slices.Contains(trace, event) } + before := func(first, second string) { + i, j := slices.Index(trace, first), slices.Index(trace, second) + if j >= 0 && (i < 0 || i >= j) { + t.Errorf("%s must precede %s: %v", first, second, trace) } - snapshot := func() []string { - mu.Lock() - defer mu.Unlock() - return slices.Clone(trace) + } + for _, event := range []string{"cut", "close public", "close bridge", + "term product", "term apps", "term collector", + "kill product", "kill apps", "kill collector", + "reap product", "reap apps", "reap collector"} { + if has(event) != expected[event] { + t.Errorf("want %s=%t: %v", event, expected[event], trace) } - child := func(name string) *modelChild { - return &modelChild{name, log, make(chan struct{}), nil, make(chan struct{})} + } + for _, name := range []string{"product", "apps"} { + for _, first := range []string{"hold apps", "hold collector", "watchdogs off", "announcer off", "close listener", "cut", "close public"} { + before(first, "term "+name) } - product, apps, collector := child("product"), child("apps"), child("collector") - product.err = errors.New("product failed") - public := modelPublic{func() { log("close public") }} - signals := make(chan os.Signal, 2) - timers := manualTimers{timers: make(map[time.Duration]chan time.Time)} - timers.timer(cutGrace) <- time.Now() - e := newEdge() - stopped := make(chan error, 1) - draining := make(chan struct{}) - go func() { - stopped <- shutdown(shutdownParts{ - signals: signals, edge: e, public: public, product: product, productExited: product.exited, - apps: apps, collector: collector, listener: closeFunc(func() { - log("close listener") - close(draining) - }), - endWatching: func() { log("watchdogs off") }, - endAnnouncing: func() { log("announcer off") }, - endStreams: func() {}, - joinWatching: func() {}, - closeIdle: func() { - if errors.Is(context.Cause(e.base), errStopping) { - log("cut") - } - }, - closeBridge: func() { log("close bridge") }, - }, shutdownPlan{testDrain, workerdGrace, collectorGrace, timers.after, io.Discard}) - }() - signals <- syscall.SIGTERM - await(t, draining) - cutAllowed, killAllowed := false, false - check := func(trace []string) { - t.Helper() - before := func(first, second string) { - i, j := slices.Index(trace, first), slices.Index(trace, second) - if j >= 0 && (i < 0 || i >= j) { - t.Fatalf("%s must precede %s: %v", first, second, trace) - } - } - for _, name := range []string{"product", "apps"} { - for _, prerequisite := range []string{"hold apps", "hold collector", "watchdogs off", "announcer off", "close listener", "cut", "close public"} { - before(prerequisite, "term "+name) - } - before("reap "+name, "close bridge") - before("term "+name, "kill "+name) - if !killAllowed && slices.Contains(trace, "kill "+name) { - t.Fatalf("child killed before its grace: %v", trace) - } - } - before("close bridge", "term collector") - if !cutAllowed && slices.Contains(trace, "cut") { - t.Fatalf("premature cut: %v", trace) - } - if slices.Contains(trace, "kill collector") { - t.Fatalf("collector used the workerd deadline: %v", trace) - } + before("reap "+name, "close bridge") + } + before("close bridge", "term collector") + for _, name := range []string{"product", "apps", "collector"} { + before("term "+name, "kill "+name) + if has("reap "+name) && has("kill "+name) { + before("kill "+name, "reap "+name) } - for _, event := range events { - switch event { - case secondSignal: - cutAllowed = true - signals <- syscall.SIGINT - case productExit: - cutAllowed = true - close(product.exited) - case appsExit: - close(apps.exited) - case collectorExit: - close(collector.exited) - case drainTimeout: - cutAllowed = true - timers.timer(testDrain) <- time.Now() - case workerdTimeout: - killAllowed = true - timers.timer(workerdGrace) <- time.Now() - } - check(snapshot()) + } + } + + signals <- syscall.SIGTERM + check() + switch drain { + case "second-signal": + signals <- syscall.SIGINT + case "product-exit": + close(product.exited) + case "drain-timer": + timers.timer(testDrain) <- time.Now() + } + expected["cut"] = true + check() + + timers.timer(cutGrace) <- time.Now() + expected["close public"], expected["term apps"] = true, true + expected["term product"] = drain != "product-exit" + expected["reap product"] = drain == "product-exit" + check() + if !slices.Contains(trace, "term apps") && !slices.Contains(trace, "term product") { + t.Error("the model must observe at least one workerd TERM in every run") + } + + for _, event := range order { + switch event { + case "timer": + for _, name := range []string{"product", "apps"} { + // A child that has not exited must be killed when its grace expires, + // but is still not reaped until we observe its exit. + expected["kill "+name] = !expected["reap "+name] } - stopErr := await(t, stopped) - final := snapshot() - check(final) - if !slices.Contains(final, "reap collector") || stopErr == nil || - (!errors.Is(stopErr, product.err) && !slices.Contains(final, "kill product")) { - t.Fatalf("incorrect terminal outcome: trace %v, error %v", final, stopErr) + if expected["kill product"] || expected["kill apps"] { + timers.timer(workerdGrace) <- time.Now() } - }) + case "product": + if drain != "product-exit" { + close(product.exited) + } + expected["reap product"] = true + case "apps": + close(apps.exited) + expected["reap apps"] = true + } + expected["close bridge"] = expected["reap product"] && expected["reap apps"] + expected["term collector"] = expected["close bridge"] + check() + } + if collectorTimesOut { + timers.timer(collectorGrace) <- time.Now() + expected["kill collector"] = true + check() + } + close(collector.exited) + expected["reap collector"] = true + check() + select { + case err := <-stopped: + for _, name := range []string{"product", "apps", "collector"} { + reportedKilled := err != nil && strings.Contains(err.Error(), name+" did not stop") + if reportedKilled != expected["kill "+name] { + t.Errorf("want kill of %s reported=%t, got %v", name, expected["kill "+name], err) + } + } + if product.err != nil && !errors.Is(err, product.err) { + t.Errorf("product failure lost: %v", err) + } + wantFailure := product.err != nil || expected["kill product"] || expected["kill apps"] || expected["kill collector"] + if (err != nil) != wantFailure { + t.Errorf("want failure=%t, got %v", wantFailure, err) + } + default: + t.Error("shutdown did not return after the collector was reaped") } - permutations(0) } From e3ac794af9eaedd9d211b13c3be8d461ba9281de Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 13:45:48 -0800 Subject: [PATCH 6/8] Document native self-host stop timeout and watchdog grace --- apps/docs/content/run/self-host.mdx | 5 ++++- e2e/tests/docker-release-watchdog.spec.ts | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/docs/content/run/self-host.mdx b/apps/docs/content/run/self-host.mdx index 4df67839e1..e3b52b5e61 100644 --- a/apps/docs/content/run/self-host.mdx +++ b/apps/docs/content/run/self-host.mdx @@ -246,13 +246,16 @@ it. This example uses HTTPS port `4443`, for a host where another service already uses `443`: ```bash -docker run -d --name executor-v2 --init --restart unless-stopped \ +docker run -d --name executor-v2 --init --restart unless-stopped --stop-timeout 30 \ -p 127.0.0.1:4400:4400 -v executor-v2-data:/app/data \ -e BETTER_AUTH_URL=https://..ts.net:4443 \ ghcr.io/usefulsoftwareco/executor-selfhost:beta tailscale serve --bg --https=4443 http://127.0.0.1:4400 ``` +Docker's default 10-second stop timeout is shorter than the host's request drain +plus child process grace periods; keep `--stop-timeout 30` when recreating the container. + `tailscale serve status` shows the address. `BETTER_AUTH_URL` must match it exactly, including a port other than `443`; without the port, browser sign-in is rejected as an invalid origin. Changing Serve needs administrator rights on diff --git a/e2e/tests/docker-release-watchdog.spec.ts b/e2e/tests/docker-release-watchdog.spec.ts index 6d5101d08b..918fb8acdc 100644 --- a/e2e/tests/docker-release-watchdog.spec.ts +++ b/e2e/tests/docker-release-watchdog.spec.ts @@ -1,9 +1,10 @@ /** * One app isolate in a synchronous loop holds the image's only workerd JavaScript thread, so the * product and every other app stop answering, and no timeout inside workerd can fire. OSS workerd - * enforces no CPU limit, so the host's watchdog bounds the outage: it kills workerd after a run of - * failed health probes, the host exits, and the container's restart policy starts a fresh process - * in which the other apps answer again. + * enforces no CPU limit, so the host's watchdog bounds the outage: after a run of failed health + * probes, it sends SIGTERM and kills workerd five seconds later if it is still running. The host + * exits, and the container's restart policy starts a fresh process in which the other apps answer + * again. */ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; From bc5ea054d6d65b85d4b84b316b60ba188020d7f6 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 14:51:19 -0800 Subject: [PATCH 7/8] Keep Linux terminal interrupts in the native host --- apps/hosted/self-host/README.md | 1 + apps/hosted/self-host/native/process_linux.go | 5 +- .../self-host/native/process_linux_test.go | 68 +++++++++++++++++++ e2e/README.md | 4 +- e2e/tests/self-host-native-telemetry.spec.ts | 6 +- 5 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 apps/hosted/self-host/native/process_linux_test.go diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index 472c6a32b8..e74555fd17 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -410,6 +410,7 @@ ends the request drain without shortening the child processes' grace periods. Th product and apps get 10 seconds together to exit, then the telemetry collector gets 3 seconds. If a child has to be killed, the host names it in the log and exits nonzero. +On Linux, terminal Ctrl-C drains the host just like `docker stop`. ## App catalog diff --git a/apps/hosted/self-host/native/process_linux.go b/apps/hosted/self-host/native/process_linux.go index fb5032f9fb..2d3f8f17d0 100644 --- a/apps/hosted/self-host/native/process_linux.go +++ b/apps/hosted/self-host/native/process_linux.go @@ -5,7 +5,8 @@ import ( "syscall" ) -// A killed supervisor cannot leave a second process owning the product database. +// Keep terminal SIGINT at the host so it can drain, without orphaning a child if the host is killed. +// macOS keeps the shared group because it has no parent-death signal to clean up an isolated child. func configureChild(command *exec.Cmd) { - command.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} + command.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL, Setpgid: true} } diff --git a/apps/hosted/self-host/native/process_linux_test.go b/apps/hosted/self-host/native/process_linux_test.go new file mode 100644 index 0000000000..4bddcecf7a --- /dev/null +++ b/apps/hosted/self-host/native/process_linux_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestConfigureChildIsolatesTerminalSIGINT(t *testing.T) { + const helper = "EXECUTOR_TEST_SIGNAL_HOST" + if os.Getenv(helper) != "1" { + // Signal only the fixture's group, never the test runner's group. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + host := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestConfigureChildIsolatesTerminalSIGINT$") + host.Env = append(os.Environ(), helper+"=1", "GORACE=atexit_sleep_ms=0") + host.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} + if output, err := host.CombinedOutput(); err != nil { + t.Fatalf("isolated host fixture: %v\n%s", err, output) + } + return + } + + group := syscall.Getpgrp() + if group != os.Getpid() { + t.Fatal("refusing to signal a group not created for this test") + } + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, syscall.SIGINT) + defer signal.Stop(interrupt) + + child := testChildCommand("quick", "child", filepath.Join(t.TempDir(), "journal"), "", "") + configureChild(child) + ready := make(chan struct{}, 1) + child.Stdout = readyWriter{ready} + if err := child.Start(); err != nil { + t.Fatal(err) + } + p := waitProcess("child", child) + t.Cleanup(func() { + child.Process.Kill() + await(t, p.exited) + }) + await(t, ready) + pgid, err := syscall.Getpgid(child.Process.Pid) + if err != nil { + t.Fatal(err) + } + if pgid == group || pgid != child.Process.Pid { + t.Errorf("child pgid = %d, want its own group %d (host group %d)", pgid, child.Process.Pid, group) + } + if err := syscall.Kill(-group, syscall.SIGINT); err != nil { + t.Fatal(err) + } + await(t, interrupt) + if err := child.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatalf("child did not survive host-group SIGINT: %v", err) + } + await(t, p.exited) + if p.err != nil { + t.Fatalf("child did not exit cleanly after its own SIGTERM: %v", p.err) + } +} diff --git a/e2e/README.md b/e2e/README.md index ec16408745..3a879fb6e6 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -81,8 +81,8 @@ install that executable. The copy and byte comparison above stage the verified b The host supervises the collector on a private Unix socket, recorded at `.executor-telemetry` in the product data directory. Nothing listens on TCP port 4318. The scenarios check that collector ingest does not delay product health, that a crashed -collector restarts on its retained store after the supervisor's one-second backoff, and that -Ctrl-C stops it with a draining product without a restart. +collector restarts on its retained store after the supervisor's one-second backoff, and that a +collector signalled separately as the host starts draining does not restart. This needs Go and the current platform's workerd executable. The scenario owns its listeners, data directories and processes; logs remain in `.local/native-auth/` diff --git a/e2e/tests/self-host-native-telemetry.spec.ts b/e2e/tests/self-host-native-telemetry.spec.ts index ee4586aa15..513c7266ff 100644 --- a/e2e/tests/self-host-native-telemetry.spec.ts +++ b/e2e/tests/self-host-native-telemetry.spec.ts @@ -231,9 +231,9 @@ it.live(scenarios.selfHostNativeTelemetryLifecycle.title, () => (pending) => Effect.sync(() => pending.destroy()), ); yield* Effect.sleep("500 millis"); - // Ctrl-C in a terminal signals the collector and the host together. The host stops the - // supervisor immediately, so the collector stays down through the product drain, past the - // one-second restart backoff. + // Signal the collector and host separately to exercise a collector exit as the host starts + // draining. The host holds the supervisor so the collector stays down past its one-second + // restart backoff. Terminal Ctrl-C on Linux signals only the host's group. const signalled = performance.now(); yield* Effect.sync(() => { process.kill(restarted, "SIGINT"); From 3502bda12295637ed748b373b1c217d7b4671181 Mon Sep 17 00:00:00 2001 From: shuv Date: Thu, 19 Nov 2026 14:51:19 -0800 Subject: [PATCH 8/8] Document the Railway shutdown drain window --- apps/docs/content/run/self-host.mdx | 5 ++++- apps/hosted/self-host/README.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/docs/content/run/self-host.mdx b/apps/docs/content/run/self-host.mdx index e3b52b5e61..6d8a7116c7 100644 --- a/apps/docs/content/run/self-host.mdx +++ b/apps/docs/content/run/self-host.mdx @@ -226,7 +226,8 @@ still fail now and then; only three failures in a row count. Give your own probe 1. Attach a new persistent volume at `/app/data`. 2. Generate a public domain in the service's networking settings. Route it to port `8080`, which Railway supplies through `PORT`. If you override `PORT`, use that value. 3. Set the Railway healthcheck path to `/health`, with a startup timeout of 120 seconds. -4. Deploy (or redeploy if the service already started). This loads the new public domain into the server. Open that domain to create the first administrator. +4. Set `RAILWAY_DEPLOYMENT_DRAINING_SECONDS=30` (or `deploy.drainingSeconds` to `30`) so Railway gives the host time to drain before SIGKILL; the default is zero. +5. Deploy (or redeploy if the service already started). This loads the new public domain into the server. Open that domain to create the first administrator. No database service or secret variables are needed. Executor reads `RAILWAY_PUBLIC_DOMAIN` and `PORT`. The container prepares the root-owned volume, @@ -255,6 +256,8 @@ tailscale serve --bg --https=4443 http://127.0.0.1:4400 Docker's default 10-second stop timeout is shorter than the host's request drain plus child process grace periods; keep `--stop-timeout 30` when recreating the container. +On Railway, set `RAILWAY_DEPLOYMENT_DRAINING_SECONDS=30` (or `deploy.drainingSeconds` +to `30`) instead; Railway's default drain is zero seconds. `tailscale serve status` shows the address. `BETTER_AUTH_URL` must match it exactly, including a port other than `443`; without the port, browser sign-in diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index e74555fd17..97fecffc0e 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -266,6 +266,8 @@ Create an image service from `ghcr.io/usefulsoftwareco/executor-selfhost:beta`, attach a new volume at `/app/data`, and generate a public domain routed to port `8080` (Railway injects `PORT=8080`; use your own value if you override it). Set the healthcheck path to `/health` with a 120-second startup timeout. +Set `RAILWAY_DEPLOYMENT_DRAINING_SECONDS=30` (or `deploy.drainingSeconds` to `30`) +so Railway gives the host time to drain before SIGKILL; the default is zero. Deploy, or redeploy if the service already started, so it picks up the new public domain. Open the domain and create the first administrator.