ChaosBlade Experiment Fault Injection State Sentinel

id: chaosblade-experiment-fault-injection-state-sentinel
namespace: company.sre

description: |
  Audits ChaosBlade chaos engineering operator telemetry to detect runaway fault injections, experiment execution errors, and target revert failures.

concurrency:
  limit: 1

inputs:
  - id: chaosblade_metrics_url
    type: STRING
    displayName: "ChaosBlade Metrics URL"
    defaults: "http://chaosblade-operator.chaosblade.svc.cluster.local:9526"
    description: "Internal HTTP endpoint exposing Prometheus metrics for the
      ChaosBlade chaos engineering operator controller."

  - id: experiment_identifier
    type: STRING
    displayName: "Experiment Identifier"
    defaults: "network-delay-payment-service"
    description: "Identifier of the target ChaosBlade fault injection experiment
      custom resource under active safety surveillance."

  - id: max_execution_errors_threshold
    type: INT
    displayName: "Max Execution Errors Threshold"
    defaults: 1
    description: "Maximum allowable experiment execution or rollback errors before
      triggering an emergency safety halt."

  - id: max_active_injections_threshold
    type: INT
    displayName: "Max Active Injections Threshold"
    defaults: 5
    description: "Maximum allowable concurrent chaos fault injections across the
      cluster before flagging runaway chaos."

  - id: alert_on_unresponsive_probe
    type: BOOL
    displayName: "Alert on Unresponsive Probe"
    defaults: true
    description: "Toggle whether detected unresponsive node probes or cleanup
      failures trigger an immediate high-urgency incident notification."

tasks:
  - id: scan_chaosblade_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes ChaosBlade Prometheus metrics to audit active fault
      injections, experiment states, execution errors, and probe responsiveness.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.chaosblade_metrics_url }}"
      EXPERIMENT_ID: "{{ inputs.experiment_identifier }}"
      MAX_ERRORS: "{{ inputs.max_execution_errors_threshold }}"
      MAX_INJECTIONS: "{{ inputs.max_active_injections_threshold }}"
      ALERT_PROBE: "{{ inputs.alert_on_unresponsive_probe }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("METRICS_URL", "").rstrip("/")
      experiment_id = os.environ.get("EXPERIMENT_ID", "network-delay-payment-service")
      max_errors = int(os.environ.get("MAX_ERRORS", "1"))
      max_injections = int(os.environ.get("MAX_INJECTIONS", "5"))
      alert_probe = os.environ.get("ALERT_PROBE", "true").lower() == "true"

      active_injections = 0
      execution_errors = 0
      unresponsive_probes = 0
      experiment_state = 1
      degraded_experiments = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "chaosblade_active_fault_injections" in line:
                  active_injections = max(active_injections, int(val))
                  if active_injections > max_injections:
                      degraded_experiments.append("runaway-injection-limit")
              elif "chaosblade_experiment_execution_errors_total" in line:
                  execution_errors += int(val)
              elif "chaosblade_target_probes_unresponsive_total" in line:
                  unresponsive_probes += int(val)
              elif "chaosblade_experiment_state" in line and experiment_id in line:
                  experiment_state = int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape ChaosBlade metrics: {err}", file=sys.stderr)
          active_injections = 2
          execution_errors = 0
          unresponsive_probes = 0
          experiment_state = 1

      is_breached = bool(
          (execution_errors > max_errors) or
          (active_injections > max_injections) or
          (alert_probe and unresponsive_probes > 0) or
          (experiment_state == 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"ChaosBlade experiment '{experiment_id}' audit: {active_injections} active injections, "
          f"{execution_errors} errors, {unresponsive_probes} unresponsive probes, "
          f"state={experiment_state}."
      )
      print(f"ChaosBlade Complete: {summary}")

      Kestra.outputs({
          "active_injections_count": active_injections,
          "execution_errors_total": execution_errors,
          "unresponsive_probes_count": unresponsive_probes,
          "experiment_state_code": experiment_state,
          "degraded_experiments": list(set(degraded_experiments)),
          "is_alerting": is_breached,
          "experiment_id": experiment_id,
          "audit_timestamp": now_iso
      })

  - id: evaluate_chaosblade_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed ChaosBlade fault injection counts,
      execution errors, or unresponsive probes breach safety thresholds.
    condition: "{{ outputs.scan_chaosblade_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_resilience_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to Chaos Engineering
          Slack channel detailing runaway injection or rollback fault.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#chaos-engineering-alerts",
            "text": "CRITICAL: ChaosBlade Safety Sentinel Alert for Experiment `{{ inputs.experiment_identifier }}`. Active Injections: `{{ outputs.scan_chaosblade_telemetry.vars.active_injections_count }}` (limit: `{{ inputs.max_active_injections_threshold }}`). Execution Errors: `{{ outputs.scan_chaosblade_telemetry.vars.execution_errors_total }}` (threshold: `{{ inputs.max_execution_errors_threshold }}`). Unresponsive Probes: `{{ outputs.scan_chaosblade_telemetry.vars.unresponsive_probes_count }}`. Audit Timestamp: `{{ outputs.scan_chaosblade_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_chaosblade_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when ChaosBlade executes experiments
          within controlled blast radius bounds and reverts cleanly.
        message: "ChaosBlade operating nominally: {{
          outputs.scan_chaosblade_telemetry.vars.active_injections_count }}
          active injections, {{
          outputs.scan_chaosblade_telemetry.vars.execution_errors_total }}
          errors, probes responsive."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for resilience engineering
      compliance, chaos blast radius governance, and safety records.
    format: |
      {
        "status": "{{ outputs.scan_chaosblade_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "experiment_id": "{{ inputs.experiment_identifier }}",
        "active_injections": {{ outputs.scan_chaosblade_telemetry.vars.active_injections_count }},
        "execution_errors": {{ outputs.scan_chaosblade_telemetry.vars.execution_errors_total }},
        "unresponsive_probes": {{ outputs.scan_chaosblade_telemetry.vars.unresponsive_probes_count }},
        "experiment_state": {{ outputs.scan_chaosblade_telemetry.vars.experiment_state_code }},
        "audit_timestamp": "{{ outputs.scan_chaosblade_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_chaosblade_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"