MongoDB WiredTiger Cache Pressure and Dirty Page Sentinel

id: mongodb-wiredtiger-cache-pressure-and-eviction-sentinel
namespace: company.database

description: |
  Audits MongoDB WiredTiger storage engine cache pressure and dirty byte saturation
  via serverStatus metrics, flags unconstrained memory eviction risks, logs an automated
  GitHub DBA investigation ticket, and alerts the database team on Slack.

inputs:
  - id: connection_uri
    type: STRING
    displayName: MongoDB Connection URI
    defaults: "mongodb://mongo-cluster.internal.company.com:27017"

  - id: dirty_cache_percent_threshold
    type: FLOAT
    displayName: Maximum Dirty Cache Threshold (%)
    defaults: 20.0

  - id: target_repository
    type: STRING
    displayName: DBA Operations Issue Tracker
    defaults: my-org/database-operations

triggers:
  - id: periodic_wiredtiger_health_check
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/10 * * * *"
    disabled: true

tasks:
  - id: probe_wiredtiger_cache_status
    type: io.kestra.plugin.scripts.python.Script
    description: Interrogates MongoDB serverStatus WiredTiger cache metrics for
      dirty bytes and memory pressure
    containerImage: python:3.11-slim
    beforeCommands:
      - pip install pymongo kestra
    script: |
      import os
      from pymongo import MongoClient
      from kestra import Kestra

      uri = "{{ inputs.connection_uri }}"
      threshold_pct = float("{{ inputs.dirty_cache_percent_threshold }}")

      try:
          client = MongoClient(uri, serverSelectionTimeoutMS=10000)
          status = client.admin.command("serverStatus")
          client.close()
      except Exception as err:
          print(f"Failed to connect to MongoDB: {err}")
          status = {}

      wt_cache = status.get("wiredTiger", {}).get("cache", {})
      max_bytes = wt_cache.get("maximum bytes configured", 0)
      bytes_in_cache = wt_cache.get("bytes currently in the cache", 0)
      dirty_bytes = wt_cache.get("tracked dirty bytes in the cache", 0)

      # Calculate dirty cache percentage
      dirty_pct = round((dirty_bytes / max_bytes * 100.0), 2) if max_bytes > 0 else 0.0
      usage_pct = round((bytes_in_cache / max_bytes * 100.0), 2) if max_bytes > 0 else 0.0

      is_pressure_critical = dirty_pct >= threshold_pct
      print(f"WiredTiger Cache: {bytes_in_cache}/{max_bytes} ({usage_pct}%), Dirty: {dirty_bytes} ({dirty_pct}%)")

      Kestra.outputs({
          "max_bytes": max_bytes,
          "bytes_in_cache": bytes_in_cache,
          "dirty_bytes": dirty_bytes,
          "usage_pct": usage_pct,
          "dirty_pct": dirty_pct,
          "threshold_pct": threshold_pct,
          "is_pressure_critical": is_pressure_critical
      })

  - id: branch_on_cache_pressure
    type: io.kestra.plugin.core.flow.If
    description: Branches based on whether WiredTiger dirty cache exceeds the safe
      eviction threshold
    condition: "{{ outputs.probe_wiredtiger_cache_status.vars.is_pressure_critical
      == true }}"
    then:
      - id: create_dba_incident_issue
        type: io.kestra.plugin.github.issues.Create
        description: Logs a prioritized GitHub issue tracking MongoDB WiredTiger cache
          saturation
        oauthToken: "{{ secret('GITHUB_TOKEN') }}"
        repository: "{{ inputs.target_repository }}"
        title: "[DBA Alert] MongoDB WiredTiger Dirty Cache Pressure Exceeded: {{
          outputs.probe_wiredtiger_cache_status.vars.dirty_pct }}%"
        body: |
          ## 🚨 MongoDB WiredTiger Storage Engine Cache Saturation

          The database reliability sentinel detected excessive dirty byte accumulation in WiredTiger cache:
          - **Connection Host:** `{{ inputs.connection_uri }}`
          - **Dirty Cache Percentage:** **{{ outputs.probe_wiredtiger_cache_status.vars.dirty_pct }}%**
          - **Configured Threshold:** `{{ inputs.dirty_cache_percent_threshold }}%`
          - **Total Cache Utilization:** `{{ outputs.probe_wiredtiger_cache_status.vars.usage_pct }}%`

          When dirty cache exceeds 20%, WiredTiger forces foreground application threads to perform page reconciliation, causing severe query latency degradation.
          *Generated automatically by Kestra MongoDB Sentinel.*

      - id: alert_dba_slack
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-priority alert to DBA Slack channel
        url: "{{ secret('SLACK_WEBHOOK') }}"
        payload: |
          {{ {
            "text": "🚨 *MongoDB WiredTiger Cache Pressure Detected*\n*Dirty Cache:* *" ~ outputs.probe_wiredtiger_cache_status.vars.dirty_pct ~ "%* (Threshold: `" ~ inputs.dirty_cache_percent_threshold ~ "%`)\n*Total Cache Usage:* *" ~ outputs.probe_wiredtiger_cache_status.vars.usage_pct ~ "%*\n*DBA Incident:* <" ~ (outputs.create_dba_incident_issue.issueUrl ?? "#") ~ "|View GitHub Issue>"
          } | toJson }}

    else:
      - id: log_cache_healthy
        type: io.kestra.plugin.core.log.Log
        message: "MongoDB WiredTiger cache is healthy (Dirty: {{
          outputs.probe_wiredtiger_cache_status.vars.dirty_pct }}% <= {{
          inputs.dirty_cache_percent_threshold }}%)."

  - id: log_audit_complete
    type: io.kestra.plugin.core.log.Log
    description: Final status audit log recording WiredTiger cache state
    message: "WiredTiger cache audit complete. Dirty percentage: {{
      outputs.probe_wiredtiger_cache_status.vars.dirty_pct }}%."

errors:
  - id: alert_flow_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Alerts Slack when MongoDB connection or issue creation fails
    url: "{{ secret('SLACK_WEBHOOK') }}"
    payload: |
      {{ {
        "text": "⚠️ *MongoDB WiredTiger Sentinel Failed* in execution `" ~ execution.id ~ "`. Unable to query serverStatus."
      } | toJson }}