perf(ci): parallelize Railway preview deploys (#909)

.github/scripts/railway-preview-deploy.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# Deploy Railway preview app services concurrently and recover from the known
+# `railway up --ci` log-streaming false negative (railwayapp/cli#696).
+#
+# Required env:
+#   APP_SERVICES  — space-separated service names (e.g. "cio-api cio-dashboard cio-jobs")
+#   RUNNER_TEMP   — writable directory for per-service logs
+#
+# Optional env (mainly for tests):
+#   RAILWAY_DEPLOY_MAX_ATTEMPTS — poll attempts (default: 60)
+#   RAILWAY_DEPLOY_SLEEP_SECONDS — sleep between polls (default: 10)
+
+set -euo pipefail
+
+RAILWAY_DEPLOY_MAX_ATTEMPTS="${RAILWAY_DEPLOY_MAX_ATTEMPTS:-60}"
+RAILWAY_DEPLOY_SLEEP_SECONDS="${RAILWAY_DEPLOY_SLEEP_SECONDS:-10}"
+
+# Returns 0 when the deployment reached SUCCESS, 1 otherwise.
+poll_deployment_status() {
+  local service="$1"
+  local deploy_id="$2"
+  local status=""
+  local attempt
+
+  for attempt in $(seq 1 "$RAILWAY_DEPLOY_MAX_ATTEMPTS"); do
+    status=$(
+      railway deployment list --service "$service" --json 2>/dev/null |
+        jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty'
+    )
+
+    case "$status" in
+      SUCCESS)
+        echo "Deployment $deploy_id for $service actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing."
+        return 0
+        ;;
+      FAILED|CRASHED|REMOVED)
+        echo "::error::$service deployment $deploy_id did not succeed (status: $status)."
+        return 1
+        ;;
+    esac
+
+    if [ "$attempt" -eq "$RAILWAY_DEPLOY_MAX_ATTEMPTS" ]; then
+      echo "::error::$service deployment $deploy_id never reached a terminal status after $RAILWAY_DEPLOY_MAX_ATTEMPTS attempts (last seen: ${status:-unknown})."
+      return 1
+    fi
+
+    echo "Deployment $deploy_id for $service not finished yet (status: ${status:-unknown}, attempt $attempt/$RAILWAY_DEPLOY_MAX_ATTEMPTS) — retrying in ${RAILWAY_DEPLOY_SLEEP_SECONDS}s..."
+    sleep "$RAILWAY_DEPLOY_SLEEP_SECONDS"
+  done
+
+  return 1
+}
+
+# Resolve a non-zero `railway up` exit using the captured log.
+# Only the known log-streaming glitch is eligible for status polling.
+# Returns 0 on recovered SUCCESS, 1 on failure.
+resolve_railway_up_failure() {
+  local service="$1"
+  local deployment_log="$2"
+  local deploy_id
+
+  if ! grep -q "Failed to stream build logs:" "$deployment_log"; then
+    echo "::error::railway up failed for $service for a reason other than the known log-streaming glitch — not polling stale deployment status."
+    return 1
+  fi
+
+  deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$deployment_log" | head -n1)
+  if [ -z "$deploy_id" ]; then
+    echo "::error::railway up failed for $service and no deployment id could be recovered from its output — cannot verify actual status."
+    return 1
+  fi
+
+  echo "railway up reported failure for $service due to the known log-streaming glitch; checking status of deployment $deploy_id..."
+  poll_deployment_status "$service" "$deploy_id"
+}
+
+deploy_app_services_in_parallel() {
+  local -a services=()
+  local -a deployment_pids=()
+  local -a deployment_logs=()
+  local service
+  local deployment_log
+  local service_index
+  local railway_status
+  local deployment_failed=false
+
+  if [ -z "${APP_SERVICES:-}" ]; then
+    echo "::error::APP_SERVICES is required."
+    return 1
+  fi
+
+  if [ -z "${RUNNER_TEMP:-}" ]; then
+    echo "::error::RUNNER_TEMP is required."
+    return 1
+  fi
+
+  # shellcheck disable=SC2206
+  services=($APP_SERVICES)
+
+  for service in "${services[@]}"; do
+    deployment_log="$RUNNER_TEMP/railway-${service}.log"
+    echo "Starting deployment for $service..."
+    railway up --service "$service" --ci >"$deployment_log" 2>&1 &
+    deployment_pids+=("$!")
+    deployment_logs+=("$deployment_log")
+  done
+
+  for service_index in "${!services[@]}"; do
+    service="${services[$service_index]}"
+    deployment_log="${deployment_logs[$service_index]}"
+
+    if wait "${deployment_pids[$service_index]}"; then
+      railway_status=0
+    else
+      railway_status=1
+    fi
+
+    echo "::group::railway up $service"
+    cat "$deployment_log"
+
+    if [ "$railway_status" -ne 0 ]; then
+      if ! resolve_railway_up_failure "$service" "$deployment_log"; then
+        deployment_failed=true
+      fi
+    fi
+
+    echo "::endgroup::"
+  done
+
+  if [ "$deployment_failed" = true ]; then
+    echo "::error::One or more Railway app deployments failed."
+    return 1
+  fi
+}
+
+# Only auto-run when executed directly (not when sourced by tests).
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  deploy_app_services_in_parallel
+fi

.github/scripts/test-railway-preview-deploy.sh

@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# Mocked coverage for concurrent Railway preview deploys and log-stream recovery.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/railway-preview-deploy.sh
+source "$SCRIPT_DIR/railway-preview-deploy.sh"
+
+TMP_ROOT="$(mktemp -d)"
+trap 'rm -rf "$TMP_ROOT"' EXIT
+
+MOCK_BIN="$TMP_ROOT/bin"
+mkdir -p "$MOCK_BIN"
+export PATH="$MOCK_BIN:$PATH"
+
+export RUNNER_TEMP="$TMP_ROOT/runner-temp"
+mkdir -p "$RUNNER_TEMP"
+export RAILWAY_DEPLOY_MAX_ATTEMPTS=3
+export RAILWAY_DEPLOY_SLEEP_SECONDS=0
+
+write_mock_railway() {
+  local mode="$1"
+
+  cat >"$MOCK_BIN/railway" <<EOF
+#!/usr/bin/env bash
+set -euo pipefail
+
+mode="$mode"
+cmd="\${1:-}"
+
+case "\$cmd" in
+  up)
+    service=""
+    while [ \$# -gt 0 ]; do
+      case "\$1" in
+        --service)
+          service="\$2"
+          shift 2
+          ;;
+        *)
+          shift
+          ;;
+      esac
+    done
+
+    case "\$mode:\$service" in
+      success:*)
+        echo "Deploying \$service..."
+        echo "https://railway.com/project/p/service/s/deployments?id=11111111-1111-1111-1111-111111111111"
+        exit 0
+        ;;
+      stream-success:cio-api)
+        echo "Deploying cio-api..."
+        echo "https://railway.com/project/p/service/s/deployments?id=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
+        echo "Failed to stream build logs: connection reset"
+        exit 1
+        ;;
+      stream-failed:cio-api)
+        echo "Deploying cio-api..."
+        echo "https://railway.com/project/p/service/s/deployments?id=dddddddd-dddd-dddd-dddd-dddddddddddd"
+        echo "Failed to stream build logs: connection reset"
+        exit 1
+        ;;
+      other-failure:cio-api)
+        echo "Deploying cio-api..."
+        echo "error: build failed before a deployment id was published"
+        exit 1
+        ;;
+      partial-failure:cio-dashboard)
+        echo "Deploying cio-dashboard..."
+        echo "error: dashboard build failed"
+        exit 1
+        ;;
+      stream-success:*|stream-failed:*|other-failure:*|partial-failure:*)
+        echo "Deploying \$service..."
+        echo "https://railway.com/project/p/service/s/deployments?id=eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"
+        exit 0
+        ;;
+      *)
+        echo "unexpected railway up mode/service: \$mode/\$service" >&2
+        exit 99
+        ;;
+    esac
+    ;;
+  deployment)
+    service=""
+    while [ \$# -gt 0 ]; do
+      case "\$1" in
+        --service)
+          service="\$2"
+          shift 2
+          ;;
+        *)
+          shift
+          ;;
+      esac
+    done
+
+    case "\$mode:\$service" in
+      stream-success:cio-api)
+        printf '%s\\n' '[{"id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","status":"SUCCESS"}]'
+        ;;
+      stream-failed:cio-api)
+        printf '%s\\n' '[{"id":"dddddddd-dddd-dddd-dddd-dddddddddddd","status":"FAILED"}]'
+        ;;
+      *)
+        printf '%s\\n' '[]'
+        ;;
+    esac
+    ;;
+  *)
+    echo "unexpected railway command: \$cmd" >&2
+    exit 99
+    ;;
+esac
+EOF
+
+  chmod +x "$MOCK_BIN/railway"
+}
+
+assert_exit() {
+  local expected="$1"
+  local label="$2"
+  shift 2
+
+  set +e
+  "$@"
+  local actual=$?
+  set -e
+
+  if [ "$actual" -ne "$expected" ]; then
+    echo "FAIL: $label (expected exit $expected, got $actual)" >&2
+    exit 1
+  fi
+
+  echo "PASS: $label"
+}
+
+export APP_SERVICES="cio-api cio-dashboard cio-jobs"
+
+write_mock_railway success
+assert_exit 0 "concurrent success" deploy_app_services_in_parallel
+
+write_mock_railway stream-success
+assert_exit 0 "stream error followed by SUCCESS" deploy_app_services_in_parallel
+
+write_mock_railway stream-failed
+assert_exit 1 "stream error followed by FAILED" deploy_app_services_in_parallel
+
+write_mock_railway other-failure
+assert_exit 1 "non-stream failure does not poll" deploy_app_services_in_parallel
+
+write_mock_railway partial-failure
+assert_exit 1 "partial concurrent failure" deploy_app_services_in_parallel
+
+echo "All railway preview deploy recovery mocks passed."

.github/workflows/railway-preview.yml

@@ -246,62 +246,18 @@ jobs:
             sleep 5
           done
 
-      - name: Deploy each app service from this branch
+      - name: Deploy app services from this branch in parallel
         if: ${{ inputs.action == 'deploy' }}
-        # `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds,
+        # `--ci` (not `--detach`) keeps each process attached until its deployment
-        # so the URL printed later is actually live. Trade-off: builds run sequentially.
+        # succeeds or fails. Run the independent app builds concurrently, then wait for
+        # every service so the URL printed later is actually live.
         #
         # `railway up --ci` can exit 1 on a log-streaming glitch (railwayapp/cli#696) even
         # though the deploy succeeded. Only on that specific error do we poll the real
         # deployment status instead of trusting the exit code — any other failure means no
         # new deployment exists yet, so it fails immediately rather than risking a match
         # against a stale, unrelated deployment.
-        run: |
+        run: bash .github/scripts/railway-preview-deploy.sh
-          for svc in $APP_SERVICES; do
-            echo "::group::railway up $svc"
-            log_file=$(mktemp)
-            railway up --service "$svc" --ci 2>&1 | tee "$log_file"
-            railway_status=${PIPESTATUS[0]}
-            if [ "$railway_status" -ne 0 ]; then
-              if grep -q "Failed to stream build logs:" "$log_file"; then
-                deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$log_file" | head -n1)
-                if [ -z "$deploy_id" ]; then
-                  echo "::error::railway up failed for $svc and no deployment id could be recovered from its output — cannot verify actual status."
-                  rm -f "$log_file"
-                  exit 1
-                fi
-                echo "railway up reported failure for $svc due to the known log-streaming glitch; checking status of deployment $deploy_id..."
-                status=""
-                for attempt in $(seq 1 60); do
-                  status=$(railway deployment list --service "$svc" --json 2>/dev/null | jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty')
-                  case "$status" in
-                    SUCCESS)
-                      echo "Deployment $deploy_id for $svc actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing."
-                      break
-                      ;;
-                    FAILED|CRASHED|REMOVED)
-                      echo "::error::$svc deployment $deploy_id did not succeed (status: $status)."
-                      rm -f "$log_file"
-                      exit 1
-                      ;;
-                  esac
-                  if [ "$attempt" -eq 60 ]; then
-                    echo "::error::$svc deployment $deploy_id never reached a terminal status after 60 attempts (last seen: ${status:-unknown})."
-                    rm -f "$log_file"
-                    exit 1
-                  fi
-                  echo "Deployment $deploy_id for $svc not finished yet (status: ${status:-unknown}, attempt $attempt/60) — retrying in 10s..."
-                  sleep 10
-                done
-              else
-                echo "::error::railway up failed for $svc for a reason other than the known log-streaming glitch — not polling stale deployment status."
-                rm -f "$log_file"
-                exit 1
-              fi
-            fi
-            rm -f "$log_file"
-            echo "::endgroup::"
-          done
 
       - name: Setup pnpm
         if: ${{ inputs.action == 'deploy' && inputs.seed }}