Monitor ScyllaDB Multi-DC Topology and Schema Version Parity

id: scylladb-multi-dc-replication-lag-watchdog
namespace: company.team
description: |
  Catch a ScyllaDB multi-datacenter cluster drifting out of sync before application
  reads turn inconsistent. This flow queries `system.peers` from one coordinator node
  on a schedule, checks whether the peer topology responded, and routes a Slack alert
  when the check cannot confirm the coordinator sees its peers, so a disconnected
  datacenter or a stalled gossip state does not go unnoticed between deploys.

triggers:
  - id: daily_replication_watchdog
    type: io.kestra.plugin.core.trigger.Schedule
    description: Daily multi-DC topology check. Shipped disabled; enable it once the
      ScyllaDB connection secrets are configured and a manual run has been
      verified against your cluster.
    cron: "0 0 * * *"
    disabled: true

inputs:
  - id: scylla_host
    type: STRING
    displayName: "ScyllaDB Node Host"
    defaults: "scylla"
    description: Contact point of one coordinator node. This is the node the check
      queries, not the whole cluster; it must be reachable from the Kestra
      Worker.

  - id: scylla_port
    type: INT
    displayName: "ScyllaDB CQL Port"
    defaults: 9042
    description: Native CQL port the contact point listens on, defaults to 9042.

  - id: local_dc
    type: STRING
    displayName: "Local Datacenter Name"
    defaults: "datacenter1"
    description: Datacenter name used by the driver's load-balancing policy. Must
      match the dc value in that node's cassandra-rackdc.properties.

  - id: scylla_username
    type: STRING
    displayName: "ScyllaDB Username"
    defaults: "scylla"
    description: Native CQL authentication username. Paired with the SCYLLA_PASSWORD
      secret; the driver requires both or neither.

  - id: max_schema_mismatches
    type: SELECT
    displayName: "Max Allowed Schema Mismatches"
    defaults: "0"
    values:
      - "0"
      - "1"
      - "5"
    description: Reserved tolerance for how many peers may report a schema_version
      different from the majority before that counts as degraded. Not yet
      consumed by classify_outcome below, which currently classifies on query
      reachability only - wire this threshold into a schema_version comparison
      if you extend the check (see "How to extend" in this blueprint's
      description).

tasks:
  - id: check_topology_peers
    type: io.kestra.plugin.cassandra.standard.Query
    description: Read every peer this coordinator currently knows about, with each
      peer's reported datacenter, schema version, host ID, and RPC address.
      allowFailure is set so an unreachable node produces a DEGRADED
      classification below instead of failing the whole execution.
    session:
      endpoints:
        - hostname: "{{ inputs.scylla_host }}"
          port: "{{ inputs.scylla_port }}"
      localDatacenter: "{{ inputs.local_dc }}"
      username: "{{ inputs.scylla_username }}"
      password: "{{ secret('SCYLLA_PASSWORD') }}"
    cql: "SELECT peer, data_center, schema_version, host_id, rpc_address FROM
      system.peers;"
    fetchType: FETCH
    allowFailure: true

  - id: classify_outcome
    type: io.kestra.plugin.core.debug.Return
    description: Reduce the peer query to a single HEALTHY or DEGRADED status. A
      missing rows output means check_topology_peers failed (connection refused,
      auth failure, timeout) and allowFailure swallowed it rather than stopping
      the execution.
    format: "{{ outputs.check_topology_peers.rows is defined ? 'HEALTHY' :
      'DEGRADED' }}"

  - id: route_outcome
    type: io.kestra.plugin.core.flow.Switch
    description: Branch on the computed status so a healthy run only logs, and a
      degraded run pages Slack with what was being checked.
    value: "{{ outputs.classify_outcome.value }}"
    cases:
      HEALTHY:
        - id: log_healthy
          type: io.kestra.plugin.core.log.Log
          message: "ScyllaDB multi-DC node topology and schema versions are in sync."
      DEGRADED:
        - id: alert_degraded
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: Alert on-call that the coordinator node could not confirm its peer
            topology, with enough detail to start triage without re-running the
            check by hand.
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": "ScyllaDB multi-DC watchdog DEGRADED: coordinator {{ inputs.scylla_host }} (datacenter {{ inputs.local_dc }}) did not return a peer topology on its system.peers check. This usually means the node is unreachable, authentication failed, or it cannot see the rest of the cluster. Execution {{ execution.id }}."
            }
    defaults:
      - id: log_default
        type: io.kestra.plugin.core.log.Log
        message: "Completed ScyllaDB topology health check with status: {{
          outputs.classify_outcome.value }}"

errors:
  - id: alert_execution_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alert when the flow itself fails outside the handled DEGRADED path,
      for example a malformed CQL statement or a plugin error. The DEGRADED
      branch above is the expected failure path; this is the unexpected one.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "ScyllaDB multi-DC replication watchdog FAILED in execution {{ execution.id }} for coordinator {{ inputs.scylla_host }}. The topology check itself did not complete; investigate before trusting the last known status."
      }

outputs:
  - id: status
    type: STRING
    description: Final HEALTHY or DEGRADED classification for this run, ready for
      dashboards or a downstream flow.
    value: "{{ outputs.classify_outcome.value }}"