AWS CloudWatch Log Group Retention Sentinel

id: aws-cloudwatch-log-group-retention-sentinel
namespace: company.team
description: |
  AWS CloudWatch log retention and FinOps governance sentinel. Scans regional CloudWatch Log Groups
  to identify groups with missing retention periods ('Never Expire') and alerts FinOps teams on Slack.

concurrency:
  limit: 1

inputs:
  - id: aws_region
    type: STRING
    displayName: AWS Region
    defaults: "us-east-1"
    description: "Target AWS region to inspect for CloudWatch Log Groups."

  - id: recommended_retention_days
    type: INT
    displayName: Recommended Retention Days
    defaults: 30
    description: "Standard compliance retention period in days to assign to
      unmanaged log groups."

  - id: log_storage_cost_per_gb_month
    type: FLOAT
    displayName: Monthly Storage Cost per GB
    defaults: 0.03
    description: "Estimated AWS CloudWatch standard log storage pricing per gigabyte
      per month."

  - id: slack_channel
    type: STRING
    displayName: Slack Channel
    defaults: "#finops-alerts"
    description: "Slack channel destination for unmanaged log storage cost alerts."

triggers:
  - id: weekly_log_retention_audit
    type: io.kestra.plugin.core.trigger.Schedule
    description: "Audits CloudWatch log retention policies weekly on Monday at 08:00 UTC."
    cron: "0 8 * * 1"

tasks:
  - id: scan_log_group_retention
    type: io.kestra.plugin.scripts.python.Script
    description: "Queries AWS CloudWatch Logs API via boto3 to detect log groups
      configured with indefinite retention."
    containerImage: python:3.11-slim
    taskRunner:
      type: io.kestra.plugin.scripts.runner.docker.Docker
    outputFiles:
      - cloudwatch_retention_report.json
    script: |
      import os
      import json
      from kestra import Kestra

      region = "{{ inputs.aws_region }}"
      rec_days = int("{{ inputs.recommended_retention_days }}")
      cost_per_gb = float("{{ inputs.log_storage_cost_per_gb_month }}")

      # Simulated discovery representing boto3 logs.describe_log_groups() response
      mock_groups = [
          {
              "logGroupName": "/aws/lambda/payment-webhook-processor",
              "storedBytes": 145000000000,  # ~145 GB
              "retentionInDays": None,      # Never expire
              "arn": f"arn:aws:logs:{region}:123456789012:log-group:/aws/lambda/payment-webhook-processor:*"
          },
          {
              "logGroupName": "/aws/ecs/catalog-service",
              "storedBytes": 42000000000,   # ~42 GB
              "retentionInDays": None,      # Never expire
              "arn": f"arn:aws:logs:{region}:123456789012:log-group:/aws/ecs/catalog-service:*"
          }
      ]

      unmanaged_groups = []
      total_wasted_bytes = 0

      for group in mock_groups:
          if group.get("retentionInDays") is None:
              stored_bytes = group.get("storedBytes", 0)
              stored_gb = round(stored_bytes / (1024 ** 3), 2)
              est_monthly_cost = round(stored_gb * cost_per_gb, 2)
              total_wasted_bytes += stored_bytes

              unmanaged_groups.append({
                  "log_group_name": group["logGroupName"],
                  "stored_gb": stored_gb,
                  "estimated_monthly_cost_usd": est_monthly_cost
              })

      has_unmanaged = len(unmanaged_groups) > 0
      total_wasted_gb = round(total_wasted_bytes / (1024 ** 3), 2)
      total_monthly_waste = round(total_wasted_gb * cost_per_gb, 2)
      top_group = unmanaged_groups[0]["log_group_name"] if unmanaged_groups else "none"

      report = {
          "region": region,
          "unmanaged_groups_count": len(unmanaged_groups),
          "total_wasted_gb": total_wasted_gb,
          "estimated_monthly_spend_usd": total_monthly_waste,
          "groups": unmanaged_groups
      }

      with open("cloudwatch_retention_report.json", "w") as f:
          json.dump(report, f, indent=2)

      Kestra.outputs({
          "has_unmanaged_groups": has_unmanaged,
          "unmanaged_count": len(unmanaged_groups),
          "total_wasted_gb": total_wasted_gb,
          "total_monthly_spend_usd": total_monthly_waste,
          "top_log_group": top_group
      })

  - id: check_unmanaged_log_groups
    type: io.kestra.plugin.core.flow.If
    description: "Evaluates whether any CloudWatch Log Groups lack expiration policies."
    condition: "{{ outputs.scan_log_group_retention.vars.has_unmanaged_groups }}"
    then:
      - id: notify_slack_finops
        type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
        description: "Dispatches actionable cost optimization card to FinOps team with
          remediation command."
        url: "{{ secret('SLACK_WEBHOOK_URL') }}"
        payload: |
          {
            "channel": "{{ inputs.slack_channel }}",
            "text": "*FINOPS ALERT: Unmanaged CloudWatch Log Groups Detected*\n• *Region*: `{{ inputs.aws_region }}`\n• *Unmanaged Groups*: {{ outputs.scan_log_group_retention.vars.unmanaged_count }} groups set to 'Never Expire'\n• *Accumulated Storage*: {{ outputs.scan_log_group_retention.vars.total_wasted_gb }} GB (~${{ outputs.scan_log_group_retention.vars.total_monthly_spend_usd }}/month)\n• *Primary Offender*: `{{ outputs.scan_log_group_retention.vars.top_log_group }}`\n• *Operational Impact*: Indefinite retention causes AWS CloudWatch storage costs to compound continuously without operational need.\n• *Recommended Remediation*: Apply a {{ inputs.recommended_retention_days }}-day retention policy via AWS CLI: `aws logs put-retention-policy --log-group-name {{ outputs.scan_log_group_retention.vars.top_log_group }} --retention-in-days {{ inputs.recommended_retention_days }}`."
          }

    else:
      - id: log_compliant_retention
        type: io.kestra.plugin.core.log.Log
        description: "Logs nominal status confirmation across audited CloudWatch log
          groups."
        message: "CloudWatch log retention audit compliant. All log groups in region '{{
          inputs.aws_region }}' have active retention policies configured."

  - id: export_retention_manifest
    type: io.kestra.plugin.core.debug.Return
    description: "Outputs structured JSON compliance manifest for FinOps reporting."
    format: |
      {
        "pipeline": "aws-cloudwatch-log-group-retention-sentinel",
        "audited_at": "{{ execution.startDate }}",
        "region": "{{ inputs.aws_region }}",
        "unmanaged_groups_count": {{ outputs.scan_log_group_retention.vars.unmanaged_count }},
        "total_wasted_gb": {{ outputs.scan_log_group_retention.vars.total_wasted_gb }},
        "status": "{{ outputs.scan_log_group_retention.vars.has_unmanaged_groups ? 'ACTION_REQUIRED' : 'COMPLIANT' }}"
      }