KeyDB Active-Replica Multi-Master Peering Sentinel

id: keydb-active-replica-multimaster-sentinel
namespace: company.data

inputs:
  - id: keydb_host
    type: STRING
    displayName: KeyDB Host
    defaults: "keydb-node-1.internal"
    description: Hostname or IP address of the primary KeyDB instance to interrogate.

  - id: keydb_port
    type: INT
    displayName: KeyDB Port
    defaults: 6379
    description: Port number for the Redis wire protocol interface on KeyDB (default 6379).

  - id: peer_host
    type: STRING
    displayName: Active Replica Peer Host
    defaults: "keydb-node-2.internal"
    description: Hostname of the companion active-replica peer node used for
      automated re-pairing.

  - id: peer_port
    type: INT
    displayName: Active Replica Peer Port
    defaults: 6379
    description: Port number of the companion active-replica peer node (default 6379).

  - id: use_password
    type: BOOL
    displayName: Use Password Authentication
    defaults: false
    description: Authenticate using the KEYDB_PASSWORD secret when connecting via RedisCLI.

  - id: slack_channel
    type: STRING
    displayName: Slack Channel
    defaults: "#data-platform-alerts"
    description: Slack channel destination for operational replication incident cards.

  - id: auto_reconnect
    type: BOOL
    displayName: Auto Reconnect Dropped Peering
    defaults: true
    description: Dispatch REPLICAOF command to re-establish active replication when
      links enter down state.

  - id: dry_run
    type: BOOL
    displayName: Dry Run Mode
    defaults: false
    description: When true, audits replication health and dispatches notifications
      but skips remediation commands.

triggers:
  - id: every_fifteen_minutes
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"

tasks:
  - id: probe_replication_info
    type: io.kestra.plugin.redis.cli.RedisCLI
    host: "{{ inputs.keydb_host }}"
    port: "{{ inputs.keydb_port }}"
    password: "{{ inputs.use_password ? secret('KEYDB_PASSWORD') : null }}"
    commands:
      - INFO replication

  - id: route_by_replication_health
    type: io.kestra.plugin.core.flow.Switch
    value: >-
      {% if outputs.probe_replication_info.vars.command_1 is null %}
      PROBE_FAILED {% else %}
        {% set info = outputs.probe_replication_info.vars.command_1 %}
        {% if "master_link_status:down" in info or "master_global_link_status:down" in info %}
        CRITICAL
        {% elseif "master_sync_in_progress:1" in info %}
        WARNING
        {% elseif "role:active-replica" in info or "role:master" in info %}
        HEALTHY
        {% else %}
        UNKNOWN
        {% endif %}
      {% endif %}
    cases:
      CRITICAL:
        - id: alert_critical_slack
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "{{ inputs.slack_channel }}",
              "text": "[CRITICAL] KeyDB Active-Replica Peering Link Down - Multi-Master State Diverging",
              "attachments": [
                {
                  "color": "#D32F2F",
                  "title": "KeyDB Active-Replica Replication Link Failure",
                  "fields": [
                    {
                      "title": "Local Node",
                      "value": "{{ inputs.keydb_host }}:{{ inputs.keydb_port }}",
                      "short": true
                    },
                    {
                      "title": "Companion Peer",
                      "value": "{{ inputs.peer_host }}:{{ inputs.peer_port }}",
                      "short": true
                    },
                    {
                      "title": "Severity",
                      "value": "CRITICAL - Active-Replica Link Down",
                      "short": true
                    },
                    {
                      "title": "Remediation Status",
                      "value": "{{ (inputs.auto_reconnect and not inputs.dry_run) ? 'Dispatching REPLICAOF Reconnection' : 'Remediation Skipped (Dry Run or Disabled)' }}",
                      "short": true
                    }
                  ],
                  "footer": "Kestra KeyDB Sentinel | Execution ID: {{ execution.id }}"
                }
              ]
            }

        - id: check_remediation
          type: io.kestra.plugin.core.flow.If
          condition: "{{ inputs.auto_reconnect == true and inputs.dry_run == false }}"
          then:
            - id: execute_reconnect
              type: io.kestra.plugin.redis.cli.RedisCLI
              host: "{{ inputs.keydb_host }}"
              port: "{{ inputs.keydb_port }}"
              password: "{{ inputs.use_password ? secret('KEYDB_PASSWORD') : null }}"
              commands:
                - REPLICAOF {{ inputs.peer_host }} {{ inputs.peer_port }}

            - id: notify_remediation_slack
              type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
              url: "{{ secret('SLACK_WEBHOOK_URL') }}"
              payload: |
                {
                  "channel": "{{ inputs.slack_channel }}",
                  "text": "[REMEDIATION] KeyDB REPLICAOF command dispatched to re-establish active-replica peering link."
                }

      WARNING:
        - id: alert_warning_slack
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "{{ inputs.slack_channel }}",
              "text": "[WARNING] KeyDB Active-Replica Sync In Progress - Initial Handshake or Resync Active",
              "attachments": [
                {
                  "color": "#F57C00",
                  "title": "KeyDB Replication Synchronization Warning",
                  "fields": [
                    {
                      "title": "Local Node",
                      "value": "{{ inputs.keydb_host }}:{{ inputs.keydb_port }}",
                      "short": true
                    },
                    {
                      "title": "Status",
                      "value": "master_sync_in_progress: 1",
                      "short": true
                    },
                    {
                      "title": "Severity",
                      "value": "WARNING - Background Syncing",
                      "short": true
                    },
                    {
                      "title": "Auto Reconnect",
                      "value": "{{ inputs.auto_reconnect ? 'Enabled' : 'Disabled' }}",
                      "short": true
                    }
                  ],
                  "footer": "Kestra KeyDB Sentinel | Execution ID: {{ execution.id }}"
                }
              ]
            }

      PROBE_FAILED:
        - id: alert_probe_failure
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "channel": "{{ inputs.slack_channel }}",
              "text": "[CRITICAL] KeyDB Sentinel Probe Failed - Unable to execute INFO replication via RedisCLI. Verify network routing and credentials."
            }

      HEALTHY:
        - id: log_replication_healthy
          type: io.kestra.plugin.core.log.Log
          message: "KeyDB active-replica replication health is normal. Peering link status
            is UP."

  - id: log_audit_summary
    type: io.kestra.plugin.core.log.Log
    message: "KeyDB replication health audit completed. Status: {{
      outputs.route_by_replication_health.value ?? 'PROCESSED' }}."

errors:
  - id: on_flow_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "channel": "{{ inputs.slack_channel }}",
        "text": "[ERROR] KeyDB Active-Replica Sentinel flow execution aborted unexpectedly. Execution ID: {{ execution.id }}."
      }