id: railway-deployment-crash-loop-and-memory-oom-sentinel
namespace: company.cloud
description: |
Audit Railway application service deployments on a schedule to detect crashed containers,
memory OOM exits, and failed build pipelines before service degradation impacts users.
inputs:
- id: railway_project_id
type: STRING
defaults: "019283a1-b2c3-7d4e-5f6a-7b8c9d0e1f2a"
displayName: "Railway Project ID"
description: "UUID of the target Railway project (found in Railway Project Settings)."
- id: environment_name
type: STRING
defaults: "production"
displayName: "Target Environment"
description: "Deployment environment to inspect (e.g. production, staging)."
- id: notification_channel
type: STRING
defaults: "#devops-alerts"
displayName: "Slack Notification Channel"
description: "Target Slack channel for deployment incident alerts and recovery
commands."
- id: remediation_mode
type: SELECT
values:
- "ALERT_ONLY"
- "LOG_RESTART_GUIDANCE"
defaults: "ALERT_ONLY"
displayName: "Remediation Strategy"
description: "Response strategy on detected failure: dispatch Slack alert only
or log CLI rollback directives."
triggers:
- id: periodic_railway_audit
type: io.kestra.plugin.core.trigger.Schedule
cron: "0 */2 * * *"
disabled: true
inputs:
railway_project_id: "019283a1-b2c3-7d4e-5f6a-7b8c9d0e1f2a"
environment_name: "production"
notification_channel: "#devops-alerts"
remediation_mode: "ALERT_ONLY"
tasks:
- id: fetch_railway_deployments
type: io.kestra.plugin.core.http.Request
description: "Query Railway GraphQL API v2 to retrieve active deployment
statuses across all project services."
uri: "https://backboard.railway.app/graphql/v2"
method: POST
headers:
Authorization: "Bearer {{ secret('RAILWAY_API_TOKEN') }}"
Content-Type: "application/json"
contentType: "application/json"
body: |
{
"query": "query GetProjectDeployments($projectId: String!) { project(id: $projectId) { services { edges { node { id name deployments(first: 3) { edges { node { id status createdAt meta } } } } } } } }",
"variables": {
"projectId": "{{ inputs.railway_project_id }}"
}
}
- id: parse_deployment_health
type: io.kestra.plugin.scripts.python.Script
description: "Parse GraphQL response, detect crashed or failed deployment
statuses, and isolate offending services."
containerImage: python:3.11-slim
env:
RAW_GRAPHQL: "{{ outputs.fetch_railway_deployments.body }}"
PROJECT_ID: "{{ inputs.railway_project_id }}"
ENV_NAME: "{{ inputs.environment_name }}"
script: |
import os
import json
raw_payload = os.environ.get("RAW_GRAPHQL", "{}")
project_id = os.environ.get("PROJECT_ID", "")
env_name = os.environ.get("ENV_NAME", "production")
try:
data = json.loads(raw_payload) if isinstance(raw_payload, str) and raw_payload.startswith("{") else {}
except Exception:
data = {}
services = data.get("data", {}).get("project", {}).get("services", {}).get("edges", [])
failed_deployments = []
healthy_services = 0
for s_edge in services:
svc = s_edge.get("node", {})
svc_name = svc.get("name", "unknown")
deployments = svc.get("deployments", {}).get("edges", [])
if deployments:
latest = deployments[0].get("node", {})
status = latest.get("status", "UNKNOWN").upper()
if status in ["CRASHED", "FAILED", "NEEDS_APPROVAL", "REMOVED"]:
failed_deployments.append({
"service_name": svc_name,
"deployment_id": latest.get("id"),
"status": status,
"created_at": latest.get("createdAt")
})
elif status in ["SUCCESS", "ACTIVE"]:
healthy_services += 1
classification = "DEPLOYMENTS_DEGRADED" if len(failed_deployments) > 0 else "HEALTHY"
outputs = {
"status": classification,
"total_services": len(services),
"healthy_services": healthy_services,
"failed_count": len(failed_deployments),
"failed_deployments": failed_deployments
}
print(f"Railway Audit: {len(failed_deployments)} failed across {len(services)} services - Status: {classification}")
print("::" + json.dumps({"outputs": outputs}) + "::")
- id: evaluate_deployment_status
type: io.kestra.plugin.core.flow.Switch
description: "Route workflow execution based on whether any Railway services
have crashed deployments."
value: "{{ outputs.parse_deployment_health.vars.status }}"
cases:
"HEALTHY":
- id: log_deployments_healthy
type: io.kestra.plugin.core.log.Log
description: "Record healthy status when all project services report active,
passing deployments."
message: "Railway deployments are healthy for project '{{
inputs.railway_project_id }}' in '{{ inputs.environment_name }}'. {{
outputs.parse_deployment_health.vars.healthy_services }}/{{
outputs.parse_deployment_health.vars.total_services }} services
online."
"DEPLOYMENTS_DEGRADED":
- id: alert_crashed_deployment
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatch urgent operational notification to Slack detailing
degraded Railway services."
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{
"channel": "{{ inputs.notification_channel }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "🚨 Railway Deployment Failure Alert"
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Project ID:*\n`{{ inputs.railway_project_id }}`"
},
{
"type": "mrkdwn",
"text": "*Environment:*\n`{{ inputs.environment_name }}`"
},
{
"type": "mrkdwn",
"text": "*Offending Service:*\n🔥 *{{ outputs.parse_deployment_health.vars.failed_deployments[0]['service_name'] }}*"
},
{
"type": "mrkdwn",
"text": "*Status:*\n`{{ outputs.parse_deployment_health.vars.failed_deployments[0]['status'] }}`"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "⚠️ *Action Required:* Service `{{ outputs.parse_deployment_health.vars.failed_deployments[0]['service_name'] }}` failed deployment `{{ outputs.parse_deployment_health.vars.failed_deployments[0]['deployment_id'] }}`. Inspect container build logs or redeploy previous revision."
}
}
]
}
- id: log_remediation_guidance
type: io.kestra.plugin.core.log.Log
description: "Log diagnostic and rollback guidance for DevOps engineers to
remediate crashed Railway containers."
message: "Railway Remediation: Service '{{
outputs.parse_deployment_health.vars.failed_deployments[0]['service\
_name'] }}' is in state '{{
outputs.parse_deployment_health.vars.failed_deployments[0]['status']
}}'. Check OOM limits in railway.json or execute 'railway redeploy
--service {{
outputs.parse_deployment_health.vars.failed_deployments[0]['service\
_name'] }}'. Mode: {{ inputs.remediation_mode }}."
defaults:
- id: log_unknown_state
type: io.kestra.plugin.core.log.Log
description: "Fallback logger for unclassified deployment audit states."
message: "Railway deployment audit completed with unclassified metrics."
- id: log_audit_completion
type: io.kestra.plugin.core.log.Log
description: "Record completion of the Railway deployment sentinel inspection run."
message: "Railway deployment sentinel audit pass completed for project '{{
inputs.railway_project_id }}'. Degraded services: {{
outputs.parse_deployment_health.vars.failed_count ?? 0 }}."
errors:
- id: alert_on_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: "Dispatch urgent Slack notification if flow execution encounters an
unhandled exception."
url: "{{ secret('SLACK_WEBHOOK') }}"
payload: |
{"text": "Flow execution failed: railway-deployment-crash-loop-and-memory-oom-sentinel encountered an unhandled error during execution {{ execution.id }}."}