fix: flakiness in railway deployment logs (#896)
* fix: handle flakiness in railway deployment logs by verifying actual deployment status * fix: improve handling of railway deployment log-streaming errors to ensure accurate deployment status * fix: enhance deployment status verification to address log-streaming glitches in railway * fix: allow running of railway workflow for forked version * Apply suggestions from code review Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: Rotimi Best <best@classroomio.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Rotimi Best
parent
0be63fe558
commit
d3a0e01f2b
@@ -22,6 +22,11 @@ name: Railway Preview Environment
|
|||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
|
repo:
|
||||||
|
description: "Repo to deploy from (leave blank for classroomio/classroomio; set to e.g. Arun5768/classroomio for a fork)"
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: ""
|
||||||
branch:
|
branch:
|
||||||
description: "Branch (or PR head ref) to deploy into the preview env"
|
description: "Branch (or PR head ref) to deploy into the preview env"
|
||||||
required: true
|
required: true
|
||||||
@@ -61,6 +66,7 @@ jobs:
|
|||||||
- name: Checkout target branch
|
- name: Checkout target branch
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
|
repository: ${{ inputs.repo != '' && inputs.repo || github.repository }}
|
||||||
ref: ${{ inputs.branch }}
|
ref: ${{ inputs.branch }}
|
||||||
|
|
||||||
- name: Install Railway CLI
|
- name: Install Railway CLI
|
||||||
@@ -244,10 +250,56 @@ jobs:
|
|||||||
if: ${{ inputs.action == 'deploy' }}
|
if: ${{ inputs.action == 'deploy' }}
|
||||||
# `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds,
|
# `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds,
|
||||||
# so the URL printed later is actually live. Trade-off: builds run sequentially.
|
# so the URL printed later is actually live. Trade-off: builds run sequentially.
|
||||||
|
#
|
||||||
|
# `railway up --ci` can exit 1 on a log-streaming glitch (railwayapp/cli#696) even
|
||||||
|
# though the deploy succeeded. Only on that specific error do we poll the real
|
||||||
|
# deployment status instead of trusting the exit code — any other failure means no
|
||||||
|
# new deployment exists yet, so it fails immediately rather than risking a match
|
||||||
|
# against a stale, unrelated deployment.
|
||||||
run: |
|
run: |
|
||||||
for svc in $APP_SERVICES; do
|
for svc in $APP_SERVICES; do
|
||||||
echo "::group::railway up $svc"
|
echo "::group::railway up $svc"
|
||||||
railway up --service "$svc" --ci
|
log_file=$(mktemp)
|
||||||
|
railway up --service "$svc" --ci 2>&1 | tee "$log_file"
|
||||||
|
railway_status=${PIPESTATUS[0]}
|
||||||
|
if [ "$railway_status" -ne 0 ]; then
|
||||||
|
if grep -q "Failed to stream build logs:" "$log_file"; then
|
||||||
|
deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$log_file" | head -n1)
|
||||||
|
if [ -z "$deploy_id" ]; then
|
||||||
|
echo "::error::railway up failed for $svc and no deployment id could be recovered from its output — cannot verify actual status."
|
||||||
|
rm -f "$log_file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "railway up reported failure for $svc due to the known log-streaming glitch; checking status of deployment $deploy_id..."
|
||||||
|
status=""
|
||||||
|
for attempt in $(seq 1 60); do
|
||||||
|
status=$(railway deployment list --service "$svc" --json 2>/dev/null | jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty')
|
||||||
|
case "$status" in
|
||||||
|
SUCCESS)
|
||||||
|
echo "Deployment $deploy_id for $svc actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing."
|
||||||
|
break
|
||||||
|
;;
|
||||||
|
FAILED|CRASHED|REMOVED)
|
||||||
|
echo "::error::$svc deployment $deploy_id did not succeed (status: $status)."
|
||||||
|
rm -f "$log_file"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
if [ "$attempt" -eq 60 ]; then
|
||||||
|
echo "::error::$svc deployment $deploy_id never reached a terminal status after 60 attempts (last seen: ${status:-unknown})."
|
||||||
|
rm -f "$log_file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Deployment $deploy_id for $svc not finished yet (status: ${status:-unknown}, attempt $attempt/60) — retrying in 10s..."
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
else
|
||||||
|
echo "::error::railway up failed for $svc for a reason other than the known log-streaming glitch — not polling stale deployment status."
|
||||||
|
rm -f "$log_file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
rm -f "$log_file"
|
||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user