Netdata Node Health and System Alarm Sentinel

id: netdata-node-health-and-system-alarm-sentinel
namespace: company.infrastructure

description: |
  Continuously monitors Netdata real-time host metrics agents, audits active critical and warning alarms for CPU, memory, and disk saturation, and dispatches actionable Slack incident alerts upon node resource degradation.

inputs:
  - id: netdata_url
    type: STRING
    displayName: Netdata Endpoint URL
    defaults: "http://localhost:19999"
    description: "Base HTTP URL of the Netdata monitoring agent."

  - id: target_hostname
    type: STRING
    displayName: Target Hostname
    defaults: "prod-node-01"
    description: "Expected hostname identifier of the audited node."

  - id: slack_channel
    type: STRING
    displayName: SRE Alert Channel
    defaults: "#infra-alerts"
    description: "Slack channel for routing host alarm and node saturation alerts."

triggers:
  - id: scheduled_sentinel_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "0 */1 * * *"
    disabled: true
    inputs:
      netdata_url: "http://localhost:19999"
      target_hostname: "prod-node-01"

tasks:
  - id: probe_node_info
    type: io.kestra.plugin.core.http.Request
    description: "Queries Netdata /api/v1/info to verify agent reachability, OS
      metadata, and system uptime."
    uri: "{{ inputs.netdata_url }}/api/v1/info"
    method: GET

  - id: probe_active_alarms
    type: io.kestra.plugin.core.http.Request
    description: "Queries Netdata /api/v1/alarms?active to retrieve currently
      triggered critical and warning alarms."
    uri: "{{ inputs.netdata_url }}/api/v1/alarms?active"
    method: GET

  - id: evaluate_node_alarms
    type: io.kestra.plugin.core.flow.Switch
    description: "Evaluates Netdata agent responsiveness and active alarms using
      Pebble conditionals."
    value: |
      {% if outputs.probe_node_info.code != 200 %}
      AGENT_UNREACHABLE
      {% elseif outputs.probe_active_alarms.code != 200 %}
      ALARMS_API_ERROR
      {% elseif outputs.probe_active_alarms.body.alarms is not empty %}
      ALARMS_ACTIVE
      {% else %}
      COMPLIANT
      {% endif %}
    cases:
      COMPLIANT:
        - id: log_compliant_state
          type: io.kestra.plugin.core.log.Log
          level: INFO
          message: "Netdata agent at {{ inputs.netdata_url }} (host: {{
            outputs.probe_node_info.body.hostname |
            default(inputs.target_hostname) }}) is healthy with zero active
            critical or warning alarms."

      ALARMS_ACTIVE:
        - id: alert_active_alarms
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "attachments": [
                {
                  "color": "#D32F2F",
                  "title": "Netdata Host Alarm Triggered",
                  "text": "CRITICAL: Active system alarms detected on host `{{ outputs.probe_node_info.body.hostname | default(inputs.target_hostname) }}` via Netdata at `{{ inputs.netdata_url }}`.\nResource saturation or hardware thresholds have been exceeded. Review node dashboards immediately.",
                  "footer": "Kestra Sentinel | Infrastructure Reliability"
                }
              ]
            }
        - id: log_alarms_triggered
          type: io.kestra.plugin.core.log.Log
          level: WARN
          message: "CRITICAL: Active alarms detected on node {{
            outputs.probe_node_info.body.hostname |
            default(inputs.target_hostname) }}."

      AGENT_UNREACHABLE:
        - id: alert_agent_offline
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "attachments": [
                {
                  "color": "#E65100",
                  "title": "Netdata Agent Offline",
                  "text": "WARNING: Netdata monitoring agent at `{{ inputs.netdata_url }}` is unreachable (HTTP status `{{ outputs.probe_node_info.code }}`).\nHost telemetry and automated health evaluations are offline.",
                  "footer": "Kestra Sentinel | Infrastructure Reliability"
                }
              ]
            }
        - id: log_agent_offline
          type: io.kestra.plugin.core.log.Log
          level: WARN
          message: "WARNING: Netdata agent at {{ inputs.netdata_url }} returned HTTP
            status {{ outputs.probe_node_info.code }}."

      ALARMS_API_ERROR:
        - id: alert_api_failure
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "attachments": [
                {
                  "color": "#F57C00",
                  "title": "Netdata Alarms API Error",
                  "text": "WARNING: Netdata `/api/v1/alarms` query failed with HTTP status `{{ outputs.probe_active_alarms.code }}` on `{{ inputs.netdata_url }}`.",
                  "footer": "Kestra Sentinel | Infrastructure Reliability"
                }
              ]
            }

  - id: record_audit_log
    type: io.kestra.plugin.core.log.Log
    description: "Emits a permanent execution audit log recording node health status."
    level: INFO
    message: "Completed Netdata node health audit on {{ inputs.netdata_url }} for
      host {{ outputs.probe_node_info.body.hostname |
      default(inputs.target_hostname) }}. Info code={{
      outputs.probe_node_info.code }}, Alarms code={{
      outputs.probe_active_alarms.code }}."