fix: flakiness in railway deployment logs (#896)

* fix: handle flakiness in railway deployment logs by verifying actual deployment status

* fix: improve handling of railway deployment log-streaming errors to ensure accurate deployment status

* fix: enhance deployment status verification to address log-streaming glitches in railway

* fix: allow running of railway workflow for forked version

* Apply suggestions from code review

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

---------

Co-authored-by: Rotimi Best <best@classroomio.com>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Abdulmuizz Hamzat
2026-08-14 13:48:58 +01:00
committed by GitHub
co-authored by greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> Rotimi Best
parent 0be63fe558
commit d3a0e01f2b
+53 -1
View File
@@ -22,6 +22,11 @@ name: Railway Preview Environment
on: on:
workflow_dispatch: workflow_dispatch:
inputs: inputs:
repo:
description: "Repo to deploy from (leave blank for classroomio/classroomio; set to e.g. Arun5768/classroomio for a fork)"
required: false
type: string
default: ""
branch: branch:
description: "Branch (or PR head ref) to deploy into the preview env" description: "Branch (or PR head ref) to deploy into the preview env"
required: true required: true
@@ -61,6 +66,7 @@ jobs:
- name: Checkout target branch - name: Checkout target branch
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
repository: ${{ inputs.repo != '' && inputs.repo || github.repository }}
ref: ${{ inputs.branch }} ref: ${{ inputs.branch }}
- name: Install Railway CLI - name: Install Railway CLI
@@ -244,10 +250,56 @@ jobs:
if: ${{ inputs.action == 'deploy' }} if: ${{ inputs.action == 'deploy' }}
# `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds, # `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds,
# so the URL printed later is actually live. Trade-off: builds run sequentially. # so the URL printed later is actually live. Trade-off: builds run sequentially.
#
# `railway up --ci` can exit 1 on a log-streaming glitch (railwayapp/cli#696) even
# though the deploy succeeded. Only on that specific error do we poll the real
# deployment status instead of trusting the exit code — any other failure means no
# new deployment exists yet, so it fails immediately rather than risking a match
# against a stale, unrelated deployment.
run: | run: |
for svc in $APP_SERVICES; do for svc in $APP_SERVICES; do
echo "::group::railway up $svc" echo "::group::railway up $svc"
railway up --service "$svc" --ci log_file=$(mktemp)
railway up --service "$svc" --ci 2>&1 | tee "$log_file"
railway_status=${PIPESTATUS[0]}
if [ "$railway_status" -ne 0 ]; then
if grep -q "Failed to stream build logs:" "$log_file"; then
deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$log_file" | head -n1)
if [ -z "$deploy_id" ]; then
echo "::error::railway up failed for $svc and no deployment id could be recovered from its output — cannot verify actual status."
rm -f "$log_file"
exit 1
fi
echo "railway up reported failure for $svc due to the known log-streaming glitch; checking status of deployment $deploy_id..."
status=""
for attempt in $(seq 1 60); do
status=$(railway deployment list --service "$svc" --json 2>/dev/null | jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty')
case "$status" in
SUCCESS)
echo "Deployment $deploy_id for $svc actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing."
break
;;
FAILED|CRASHED|REMOVED)
echo "::error::$svc deployment $deploy_id did not succeed (status: $status)."
rm -f "$log_file"
exit 1
;;
esac
if [ "$attempt" -eq 60 ]; then
echo "::error::$svc deployment $deploy_id never reached a terminal status after 60 attempts (last seen: ${status:-unknown})."
rm -f "$log_file"
exit 1
fi
echo "Deployment $deploy_id for $svc not finished yet (status: ${status:-unknown}, attempt $attempt/60) — retrying in 10s..."
sleep 10
done
else
echo "::error::railway up failed for $svc for a reason other than the known log-streaming glitch — not polling stale deployment status."
rm -f "$log_file"
exit 1
fi
fi
rm -f "$log_file"
echo "::endgroup::" echo "::endgroup::"
done done