Kubernetes Node Resource Pressure Sentinel

id: kubernetes-node-resource-pressure-sentinel
namespace: company.sre

description: Monitors Kubernetes cluster node conditions to proactively detect
  DiskPressure, MemoryPressure, and PIDPressure before pod evictions degrade
  cluster workloads.

concurrency:
  limit: 1

inputs:
  - id: cluster_name
    type: STRING
    defaults: production-eks-us-east-1
    description: Identifier name of the Kubernetes cluster being monitored.

  - id: alert_on_any_pressure
    type: BOOL
    defaults: true
    description: Whether to trigger an immediate Slack alert when any node reports
      resource pressure conditions.

  - id: slack_channel
    type: STRING
    defaults: "#k8s-platform-alerts"
    description: Slack channel destination for Kubernetes cluster node health alerts.

tasks:
  - id: inspect_node_conditions
    type: io.kestra.plugin.scripts.python.Script
    containerImage: python:3.11-slim
    beforeCommands:
      - pip install kubernetes
    env:
      KUBE_CONFIG_DATA: "{{ secret('KUBE_CONFIG_DATA') }}"
      CLUSTER_NAME: "{{ inputs.cluster_name }}"
    script: |
      import os
      import json
      import tempfile
      from kubernetes import client, config
      from kestra import Kestra

      kubeconfig_str = os.getenv('KUBE_CONFIG_DATA')
      with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
          f.write(kubeconfig_str)
          kubeconfig_path = f.name

      try:
          config.load_kube_config(config_file=kubeconfig_path)
          v1 = client.CoreV1Api()
          nodes = v1.list_node().items
      finally:
          if os.path.exists(kubeconfig_path):
              os.remove(kubeconfig_path)

      pressure_conditions = ['MemoryPressure', 'DiskPressure', 'PIDPressure', 'NetworkUnavailable']
      degraded_nodes = []

      for node in nodes:
          node_name = node.metadata.name
          node_status = node.status
          conditions = node_status.conditions or []

          active_pressures = []
          for cond in conditions:
              if cond.type in pressure_conditions and cond.status == 'True':
                  active_pressures.append({
                      'condition': cond.type,
                      'reason': cond.reason,
                      'message': cond.message
                  })

          if active_pressures:
              degraded_nodes.append({
                  'node_name': node_name,
                  'active_pressures': active_pressures
              })

      total_nodes = len(nodes)
      degraded_count = len(degraded_nodes)
      has_pressure = degraded_count > 0

      Kestra.outputs({
          'total_nodes_inspected': total_nodes,
          'degraded_nodes_count': degraded_count,
          'has_node_pressure': has_pressure,
          'degraded_node_names': [n['node_name'] for n in degraded_nodes]
      })

      report_payload = {
          'cluster_name': os.getenv('CLUSTER_NAME'),
          'total_nodes': total_nodes,
          'degraded_nodes_count': degraded_count,
          'degraded_nodes': degraded_nodes
      }

      with open('k8s_node_pressure_report.json', 'w') as f:
          json.dump(report_payload, f, indent=2)

    outputFiles:
      - k8s_node_pressure_report.json

  - id: check_pressure_conditions
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.inspect_node_conditions.vars.has_node_pressure and
      inputs.alert_on_any_pressure }}"
    then:
      - id: notify_slack_sre
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "{{ inputs.slack_channel }}",
            "text": "Kubernetes Node Pressure Alert: {{ outputs.inspect_node_conditions.vars.degraded_nodes_count }} node(s) experiencing resource pressure in cluster {{ inputs.cluster_name }}.\nAffected Nodes: {{ outputs.inspect_node_conditions.vars.degraded_node_names | join(', ') }}\nImmediate risk of kubelet pod eviction."
          }
    else:
      - id: log_nodes_healthy
        type: io.kestra.plugin.core.log.Log
        message: "All {{ outputs.inspect_node_conditions.vars.total_nodes_inspected }}
          Kubernetes node(s) in cluster {{ inputs.cluster_name }} are operating
          within healthy resource thresholds."

  - id: export_node_manifest
    type: io.kestra.plugin.core.execution.Labels
    labels:
      cluster_name: "{{ inputs.cluster_name }}"
      degraded_count: "{{ outputs.inspect_node_conditions.vars.degraded_nodes_count |
        string }}"
      health_status: "{{ outputs.inspect_node_conditions.vars.has_node_pressure ?
        'DEGRADED' : 'HEALTHY' }}"

triggers:
  - id: periodic_k8s_node_probe
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"