id: fastapi-synthetic-latency-slo-sentinel
namespace: company.sre

inputs:
  - id: api_endpoint_url
    type: STRING
    defaults: "https://api.github.com/zen"
    description: Target FastAPI or REST endpoint URL to evaluate for synthetic latency.

  - id: p95_latency_threshold_ms
    type: INT
    defaults: 250
    description: Maximum acceptable p95 response latency in milliseconds before
      triggering an SLO breach.

  - id: sample_iterations
    type: INT
    defaults: 5
    description: Number of consecutive synthetic probe requests to collect
      statistical latency metrics.

tasks:
  - id: run_synthetic_probes
    type: io.kestra.plugin.scripts.python.Script
    containerImage: python:3.11-slim
    beforeCommands:
      - pip install kestra requests
    script: |
      import time
      import requests
      from kestra import Kestra

      endpoint = "{{ inputs.api_endpoint_url }}"
      iterations = {{ inputs.sample_iterations }}
      latencies = []

      print(f"Executing {iterations} synthetic probes against {endpoint}...")
      for i in range(iterations):
          start_time = time.time()
          try:
              resp = requests.get(endpoint, timeout=5)
              duration_ms = (time.time() - start_time) * 1000
              latencies.append(duration_ms)
              print(f"Probe {i+1}: status={resp.status_code}, latency={duration_ms:.1f}ms")
          except Exception as e:
              latencies.append(5000.0)
              print(f"Probe {i+1} failed: {e}")

      latencies.sort()
      # Calculate p95 approximation
      idx = int(0.95 * len(latencies))
      p95_latency = latencies[min(idx, len(latencies) - 1)]
      avg_latency = sum(latencies) / len(latencies)
      threshold = {{ inputs.p95_latency_threshold_ms }}
      slo_breached = p95_latency > threshold

      print(f"Results: p95={p95_latency:.1f}ms (threshold={threshold}ms) | Breach={slo_breached}")
      Kestra.outputs({
          "p95_latency_ms": round(p95_latency, 1),
          "avg_latency_ms": round(avg_latency, 1),
          "slo_breached": slo_breached,
          "total_probes": iterations
      })

  - id: evaluate_slo_compliance
    type: io.kestra.plugin.core.flow.If
    condition: "{{ outputs.run_synthetic_probes.vars.slo_breached ?? false }}"
    then:
      - id: notify_slo_breach
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Alerts on-call SREs to degraded API latency violating service level
          objectives.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "🚨 *FastAPI Latency SLO Breach Detected:* Endpoint `{{ inputs.api_endpoint_url }}` p95 latency reached *{{ outputs.run_synthetic_probes.vars.p95_latency_ms }}ms* (Threshold: {{ inputs.p95_latency_threshold_ms }}ms). Execution `{{ execution.id }}`."
          }
    else:
      - id: notify_slo_healthy
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: Emits routine healthy status confirmation to operational monitoring
          logs.
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "text": "✅ *FastAPI Latency SLO Healthy:* Endpoint `{{ inputs.api_endpoint_url }}` running nominal at *{{ outputs.run_synthetic_probes.vars.p95_latency_ms }}ms* p95 latency."
          }

  - id: log_slo_metrics_archive
    type: io.kestra.plugin.scripts.shell.Commands
    containerImage: alpine:latest
    commands:
      - echo "Archived SLO synthetic metric snapshot for {{
        inputs.api_endpoint_url }} at $(date)"

errors:
  - id: alert_on_probe_failure
    type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
    description: Catches probe network exceptions or runner errors and alerts SREs
      immediately.
    url: "{{ secret('SLACK_WEBHOOK_URL') }}"
    payload: |
      {
        "text": "💥 *Synthetic Latency Sentinel Abort:* Failed executing probe benchmark for `{{ inputs.api_endpoint_url }}` in execution `{{ execution.id }}`. Check Kestra logs."
      }

triggers:
  - id: recurring_slo_check
    type: io.kestra.plugin.core.trigger.Schedule
    cron: "*/15 * * * *"