OpenFaaS Function Replica Saturation and Invocation Error Sentinel

id: openfaas-function-replica-and-error-sentinel
namespace: company.infrastructure

inputs:
  - id: openfaas_gateway_endpoint
    type: STRING
    displayName: OpenFaaS Gateway Endpoint
    description: Base URL of the OpenFaaS Gateway API.
    defaults: "http://localhost:8080"

  - id: max_unhealthy_functions_threshold
    type: INT
    displayName: Maximum Unhealthy Functions Threshold
    description: Number of functions allowed with degraded replicas before raising a
      critical incident alert.
    defaults: 0

  - id: send_slack_alert
    type: BOOL
    displayName: Dispatch Slack Incident Alert
    description: Whether to transmit real-time Slack notifications upon serverless
      function replica failures.
    defaults: true

triggers:
  - id: openfaas_audit_schedule
    type: io.kestra.plugin.core.trigger.Schedule
    description: Recurring audit polling OpenFaaS function replica health every 5 minutes.
    cron: "*/5 * * * *"
    disabled: true

tasks:
  - id: probe_openfaas_functions
    type: io.kestra.plugin.core.http.Request
    description: Query OpenFaaS Gateway API for deployed functions and replica
      availability metrics.
    uri: "{{ inputs.openfaas_gateway_endpoint }}/system/functions"
    method: GET
    headers:
      Accept: "application/json"
      Authorization: "Bearer {{ secret('OPENFAAS_API_TOKEN') }}"
    options:
      allowFailed: true
      timeout:
        connectTimeout: PT10S
        readIdleTimeout: PT30S
    retry:
      type: constant
      interval: PT5S
      maxAttempts: 3
    allowFailure: true

  - id: evaluate_function_replicas
    type: io.kestra.plugin.scripts.python.Script
    description: Interrogate OpenFaaS function records, detect zero-replica
      services, and classify cluster health.
    taskRunner:
      type: io.kestra.plugin.core.runner.Process
    inputFiles:
      functions.json: "{{ outputs.probe_openfaas_functions.body }}"
    script: |
      import json

      max_unhealthy = int({{ inputs.max_unhealthy_functions_threshold }})

      try:
          with open("functions.json", "r") as f:
              raw = f.read().strip()
              data = json.loads(raw) if raw else []
      except Exception:
          data = [{"name": "GATEWAY_UNREACHABLE", "replicas": 1, "availableReplicas": 0}]

      if not isinstance(data, list):
          data = []

      total_functions = len(data)
      degraded_functions = []

      for fn in data:
          name = fn.get("name", "unknown")
          replicas = int(fn.get("replicas", 0))
          available = int(fn.get("availableReplicas", 0))

          if replicas > 0 and available < replicas:
              degraded_functions.append(f"{name} ({available}/{replicas})")

      unhealthy_count = len(degraded_functions)

      if total_functions == 0 or unhealthy_count == 0:
          classification = "ALL_FUNCTIONS_HEALTHY"
      elif unhealthy_count <= max_unhealthy:
          classification = "PARTIAL_REPLICA_DEGRADATION"
      else:
          classification = "CRITICAL_FUNCTION_OUTAGE"

      degraded_summary = ", ".join(degraded_functions[:5]) if degraded_functions else "none"

      output_payload = {
          "outputs": {
              "total_functions": total_functions,
              "unhealthy_count": unhealthy_count,
              "degraded_functions": degraded_summary,
              "classification": classification
          }
      }
      print(f"::{json.dumps(output_payload)}::")

  - id: classify_serverless_status
    type: io.kestra.plugin.core.log.Log
    description: Log evaluated OpenFaaS serverless function metrics and health
      classification.
    message: |
      OpenFaaS Gateway: {{ inputs.openfaas_gateway_endpoint }}
      Total Functions: {{ outputs.evaluate_function_replicas.vars.total_functions }}
      Degraded Functions: {{ outputs.evaluate_function_replicas.vars.unhealthy_count }}
      Impacted Services: {{ outputs.evaluate_function_replicas.vars.degraded_functions }}
      Classification: {{ outputs.evaluate_function_replicas.vars.classification }}

  - id: route_replica_gate
    type: io.kestra.plugin.core.flow.Switch
    description: Route flow execution based on OpenFaaS function replica health
      classification.
    value: "{{ outputs.evaluate_function_replicas.vars.classification }}"
    cases:
      ALL_FUNCTIONS_HEALTHY:
        - id: log_nominal_state
          type: io.kestra.plugin.core.log.Log
          description: Log nominal verification confirming all serverless functions have
            full replica availability.
          message: "HEALTHY: All {{
            outputs.evaluate_function_replicas.vars.total_functions }} OpenFaaS
            serverless functions are operating with optimal replica counts."

      PARTIAL_REPLICA_DEGRADATION:
        - id: log_degradation_advisory
          type: io.kestra.plugin.core.log.Log
          description: Log advisory warning when degraded function replicas are within
            acceptable threshold limits.
          message: "WARNING: Degraded OpenFaaS function replicas detected: {{
            outputs.evaluate_function_replicas.vars.degraded_functions }}."

      CRITICAL_FUNCTION_OUTAGE:
        - id: alert_slack_function_outage
          type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
          description: Dispatch urgent incident alert to platform engineering channel.
          url: "{{ secret('SLACK_WEBHOOK_URL') }}"
          payload: |
            {
              "text": ":rotating_light: *CRITICAL OpenFaaS Serverless Replica Outage Alert*\n*Gateway:* `{{ inputs.openfaas_gateway_endpoint }}`\n*Total Functions:* `{{ outputs.evaluate_function_replicas.vars.total_functions }}`\n*Degraded Services:* `{{ outputs.evaluate_function_replicas.vars.unhealthy_count }}`\n*Impacted Functions:* `{{ outputs.evaluate_function_replicas.vars.degraded_functions }}`\n*Execution:* <{{ flow.url }}|{{ execution.id }}>"
            }

        - id: remediation_outage_gate
          type: io.kestra.plugin.core.execution.Fail
          description: Fail execution to trigger incident escalation and prevent
            downstream invocation timeouts.

  - id: final_audit_log
    type: io.kestra.plugin.core.log.Log
    description: Record immutable audit log entry documenting OpenFaaS cluster
      health verification.
    message: |
      [AUDIT] OpenFaaS Function Replica Saturation and Invocation Error Sentinel completed.
      Gateway: {{ inputs.openfaas_gateway_endpoint }}
      Total: {{ outputs.evaluate_function_replicas.vars.total_functions }}
      Degraded: {{ outputs.evaluate_function_replicas.vars.unhealthy_count }}
      Verdict: {{ outputs.evaluate_function_replicas.vars.classification }}