id: k8s-off-hours-downscaler
namespace: company.platform
description: |
Scale non-production Kubernetes Deployments down to zero (or an annotated
floor) every evening, remember what they looked like, and restore them in
the morning. Each run reports replica-hours removed and an estimated saving,
skips workloads that opted out, refuses to overwrite a saved state that was
never restored, and only touches the cluster once dry-run is switched off.
labels:
team: platform
domain: finops
concurrency:
limit: 1
behavior: QUEUE
inputs:
- id: mode
type: SELECT
displayName: Action
description: down scales to the downtime floor and saves the current replicas;
up restores what the last down run saved.
values:
- down
- up
defaults: down
- id: namespaces
type: ARRAY
itemType: STRING
displayName: Namespaces
description: Namespaces whose Deployments follow the off-hours schedule. Never
include production or system namespaces here.
defaults:
- dev
- staging
- id: dry_run
type: BOOL
displayName: Dry run
description: When true the plan and the Slack digest are produced but nothing is
patched and no state is saved. Flip to false once the plan looks right.
defaults: true
- id: exclude_annotation
type: STRING
displayName: Opt-out annotation
description: Deployments carrying this annotation with the value "true" are left alone.
defaults: downscaler/exclude
- id: downtime_replicas_annotation
type: STRING
displayName: Per-workload floor annotation
description: Deployments can set this annotation to keep a non-zero number of
replicas overnight, for example a shared database proxy.
defaults: downscaler/downtime-replicas
- id: default_downtime_replicas
type: INT
displayName: Default downtime replicas
description: Replica count applied overnight to every Deployment without a
per-workload floor annotation.
defaults: 0
- id: downtime_hours
type: FLOAT
displayName: Downtime window in hours
description: Length of the off-hours window, used to turn removed replicas into
replica-hours saved.
defaults: 11
- id: blended_cost_per_replica_hour_usd
type: FLOAT
displayName: Blended cost per replica-hour (USD)
description: Rough average cost of one running replica for one hour in your
cluster. Only used for the savings estimate in the digest.
defaults: 0.05
variables:
kv_prefix: downscaler_state
triggers:
- id: scale_down_evening
type: io.kestra.plugin.core.trigger.Schedule
description: Weekday evening scale-down. Enable together with scale_up_morning
once a dry run looks right.
cron: "0 20 * * 1-5"
timezone: Europe/Paris
disabled: true
inputs:
mode: down
dry_run: false
- id: scale_up_morning
type: io.kestra.plugin.core.trigger.Schedule
description: Weekday morning restore. Keep this enabled whenever
scale_down_evening is enabled.
cron: "0 7 * * 1-5"
timezone: Europe/Paris
disabled: true
inputs:
mode: up
dry_run: false
- id: on_demand
type: io.kestra.plugin.core.trigger.Webhook
description: Manual override, for example "wake staging up now, we have a demo".
Send mode and dry_run as JSON body or query parameters.
key: replace-me-with-a-long-random-key
tasks:
- id: inventory
type: io.kestra.plugin.core.flow.Loop
description: Read the Deployments in every target namespace, plus the replica
state saved by the last down run, and expose both per iteration.
values: "{{ inputs.namespaces }}"
concurrencyLimit: 3
fetchType: FETCH
outputs:
- id: namespace
type: STRING
value: "{{ item.value }}"
- id: metadata_items
type: JSON
value: "{{ (outputs.deployments.metadataItems ?? []) | toJson }}"
- id: status_items
type: JSON
value: "{{ (outputs.deployments.statusItems ?? []) | toJson }}"
- id: saved_state
type: STRING
value: "{{ (outputs.saved_state.value ?? '') | toJson }}"
tasks:
- id: deployments
type: io.kestra.plugin.kubernetes.kubectl.Get
description: Metadata (annotations) and status (replicas) of every Deployment in
the namespace.
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
trustCerts: true
namespace: "{{ item.value }}"
resourceType: deployments
apiGroup: apps
apiVersion: v1
fetchType: FETCH
- id: saved_state
type: io.kestra.plugin.core.kv.Get
description: Replica counts saved by the previous down run, or nothing when the
namespace is currently in its daytime shape.
key: "{{ vars.kv_prefix }}_{{ item.value }}"
errorOnMissing: false
- id: plan
type: io.kestra.plugin.scripts.python.Script
description: Turn the inventory into an explicit list of patches, the state to
remember, and the savings numbers. Nothing here talks to the cluster.
taskRunner:
type: io.kestra.plugin.core.runner.Process
env:
MODE: "{{ inputs.mode }}"
INVENTORY: "{{ outputs.inventory.outputs | toJson }}"
EXCLUDE_ANNOTATION: "{{ inputs.exclude_annotation }}"
FLOOR_ANNOTATION: "{{ inputs.downtime_replicas_annotation }}"
DEFAULT_FLOOR: "{{ inputs.default_downtime_replicas }}"
DOWNTIME_HOURS: "{{ inputs.downtime_hours }}"
COST_PER_REPLICA_HOUR: "{{ inputs.blended_cost_per_replica_hour_usd }}"
KV_PREFIX: "{{ vars.kv_prefix }}"
EXECUTION_ID: "{{ execution.id }}"
script: |
import json
import os
from datetime import datetime, timezone
mode = os.environ["MODE"]
inventory = json.loads(os.environ["INVENTORY"] or "[]")
exclude_annotation = os.environ["EXCLUDE_ANNOTATION"]
floor_annotation = os.environ["FLOOR_ANNOTATION"]
default_floor = int(os.environ["DEFAULT_FLOOR"])
downtime_hours = float(os.environ["DOWNTIME_HOURS"])
cost_per_replica_hour = float(os.environ["COST_PER_REPLICA_HOUR"])
kv_prefix = os.environ["KV_PREFIX"]
stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
actions = []
skipped = []
state_entries = []
def as_obj(value, default):
# Loop outputs may arrive as parsed JSON or as a JSON string.
if value is None or value == "":
return default
if isinstance(value, str):
try:
return json.loads(value)
except ValueError:
return default
return value
def current_replicas(status):
status = (status or {}).get("status") or {}
# status.replicas is the number of non-terminated pods the Deployment
# targets; it equals spec.replicas once a rollout has settled.
for key in ("replicas", "readyReplicas", "availableReplicas"):
if status.get(key) is not None:
return int(status[key])
return 0
for entry in inventory:
out = entry.get("outputs") or {}
namespace = out.get("namespace") or entry.get("item")
metadata_items = as_obj(out.get("metadata_items"), [])
status_items = as_obj(out.get("status_items"), [])
saved = as_obj(out.get("saved_state"), {})
if isinstance(saved, str):
saved = as_obj(saved, {})
saved_replicas = (saved or {}).get("replicas") or {}
kv_key = "{}_{}".format(kv_prefix, namespace)
if mode == "down":
if saved_replicas:
skipped.append({
"namespace": namespace,
"name": "*",
"reason": "already scaled down on {} and never restored; run mode=up first".format(saved.get("saved_at", "an earlier run")),
})
continue
remembered = {}
for meta, status in zip(metadata_items, status_items):
name = meta.get("name")
annotations = meta.get("annotations") or {}
replicas = current_replicas(status)
if str(annotations.get(exclude_annotation, "")).lower() == "true":
skipped.append({"namespace": namespace, "name": name, "reason": "opted out via annotation"})
continue
floor = default_floor
if floor_annotation in annotations:
try:
floor = max(0, int(annotations[floor_annotation]))
except ValueError:
skipped.append({"namespace": namespace, "name": name, "reason": "unreadable {} annotation".format(floor_annotation)})
continue
if replicas <= floor:
skipped.append({"namespace": namespace, "name": name, "reason": "already at or below floor ({} replicas)".format(replicas)})
continue
remembered[name] = replicas
actions.append({"namespace": namespace, "name": name, "from": replicas, "to": floor})
if remembered:
state_entries.append({
"namespace": namespace,
"key": kv_key,
"state": {"saved_at": stamp, "execution_id": os.environ["EXECUTION_ID"], "replicas": remembered},
})
else:
if not saved_replicas:
skipped.append({"namespace": namespace, "name": "*", "reason": "no saved state, nothing to restore"})
continue
live = {meta.get("name"): current_replicas(status) for meta, status in zip(metadata_items, status_items)}
for name, original in saved_replicas.items():
if name not in live:
skipped.append({"namespace": namespace, "name": name, "reason": "deployment no longer exists"})
continue
if live[name] == int(original):
skipped.append({"namespace": namespace, "name": name, "reason": "already at {} replicas".format(original)})
continue
actions.append({"namespace": namespace, "name": name, "from": live[name], "to": int(original)})
state_entries.append({"namespace": namespace, "key": kv_key})
replica_hours = 0.0
if mode == "down":
replica_hours = sum((a["from"] - a["to"]) * downtime_hours for a in actions)
estimated_savings = round(replica_hours * cost_per_replica_hour, 2)
print("mode={} actions={} skipped={} replica_hours={} estimated_savings_usd={}".format(
mode, len(actions), len(skipped), replica_hours, estimated_savings))
for a in actions:
print(" {}/{}: {} -> {}".format(a["namespace"], a["name"], a["from"], a["to"]))
for s in skipped:
print(" skip {}/{}: {}".format(s["namespace"], s["name"], s["reason"]))
print("::" + json.dumps({"outputs": {
"actions": actions,
"skipped": skipped,
"state_entries": state_entries,
"replica_hours": replica_hours,
"estimated_savings_usd": estimated_savings,
}}) + "::")
- id: apply
type: io.kestra.plugin.core.flow.If
description: Patch the cluster only when dry-run is off and the plan contains at
least one change.
condition: "{{ inputs.dry_run == false and (outputs.plan.vars.actions | length)
> 0 }}"
then:
- id: scale
type: io.kestra.plugin.core.flow.Loop
description: One strategic-merge patch per Deployment. The original replica
count is also stamped as an annotation so the cluster carries its own
audit trail.
values: "{{ outputs.plan.vars.actions }}"
concurrencyLimit: 5
tasks:
- id: patch_replicas
type: io.kestra.plugin.kubernetes.kubectl.Patch
connection:
masterUrl: "{{ secret('K8S_MASTER_URL') }}"
oauthToken: "{{ secret('K8S_TOKEN') }}"
trustCerts: true
namespace: "{{ fromJson(item.value).namespace }}"
resourceType: deployment
resourceName: "{{ fromJson(item.value).name }}"
apiGroup: apps
apiVersion: v1
patchStrategy: STRATEGIC_MERGE
patch: |
{
"metadata": {
"annotations": {
"downscaler/original-replicas": {{ inputs.mode == 'down' ? ('"' ~ fromJson(item.value).from ~ '"') : 'null' }},
"downscaler/last-action": "{{ inputs.mode }} by {{ flow.id }} {{ execution.id }}"
}
},
"spec": {
"replicas": {{ fromJson(item.value).to }}
}
}
retry:
type: constant
interval: PT10S
maxAttempts: 3
- id: remember_state
type: io.kestra.plugin.core.flow.If
description: After a down run, save the replica map per namespace in the KV
store so the morning run can restore it exactly. After an up run,
clear it.
condition: "{{ inputs.mode == 'down' }}"
then:
- id: save_state
type: io.kestra.plugin.core.flow.Loop
values: "{{ outputs.plan.vars.state_entries }}"
tasks:
- id: kv_set
type: io.kestra.plugin.core.kv.Set
key: "{{ fromJson(item.value).key }}"
value: "{{ fromJson(item.value).state | toJson }}"
kvType: JSON
overwrite: false
else:
- id: clear_state
type: io.kestra.plugin.core.flow.Loop
values: "{{ outputs.plan.vars.state_entries }}"
tasks:
- id: kv_delete
type: io.kestra.plugin.core.kv.Delete
key: "{{ fromJson(item.value).key }}"
errorOnMissing: false
else:
- id: nothing_applied
type: io.kestra.plugin.core.log.Log
message: "{{ inputs.dry_run ? 'Dry run: ' ~ (outputs.plan.vars.actions | length)
~ ' change(s) planned, nothing patched.' : 'No changes to apply.' }}"
- id: digest
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: One message per run with what changed, what was skipped and why,
and the savings estimate.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": "{{ inputs.mode == 'down' ? ':crescent_moon:' : ':sunrise:' }} *Off-hours downscaler, mode {{ inputs.mode }}{{ inputs.dry_run ? ' (dry run)' : '' }}*\nNamespaces: {{ inputs.namespaces | join(', ') }}\n{% if outputs.plan.vars.actions | length > 0 %}{% for a in outputs.plan.vars.actions %}• {{ a.namespace }}/{{ a.name }}: {{ a.from }} → {{ a.to }} replicas\n{% endfor %}{% else %}• no changes\n{% endif %}{% if outputs.plan.vars.skipped | length > 0 %}Skipped:\n{% for s in outputs.plan.vars.skipped %}• {{ s.namespace }}/{{ s.name }}: {{ s.reason }}\n{% endfor %}{% endif %}{% if inputs.mode == 'down' %}Replica-hours removed tonight: {{ outputs.plan.vars.replica_hours }} (about ${{ outputs.plan.vars.estimated_savings_usd }} at ${{ inputs.blended_cost_per_replica_hour_usd }}/replica-hour)\n{% endif %}Execution: {{ execution.id }}"
}
outputs:
- id: actions
type: JSON
description: Deployments patched (or that would be patched in dry run) with
before and after replica counts.
value: "{{ outputs.plan.vars.actions }}"
- id: skipped
type: JSON
description: Deployments left untouched and the reason for each.
value: "{{ outputs.plan.vars.skipped }}"
- id: replica_hours_removed
type: FLOAT
description: Replica-hours removed by a down run over the configured downtime window.
value: "{{ outputs.plan.vars.replica_hours }}"
- id: estimated_savings_usd
type: FLOAT
description: Replica-hours multiplied by the blended cost input.
value: "{{ outputs.plan.vars.estimated_savings_usd }}"
- id: applied
type: BOOL
description: True when patches were actually sent to the cluster.
value: "{{ inputs.dry_run == false and (outputs.plan.vars.actions | length) > 0
}}"
errors:
- id: alert_on_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: A failed evening run means workloads may be half scaled; a failed
morning run means people arrive to a sleeping environment. Either way, say
so.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"text": ":rotating_light: *Off-hours downscaler FAILED* (mode {{ inputs.mode }}, namespaces {{ inputs.namespaces | join(', ') }}). Check execution {{ execution.id }} of {{ flow.namespace }}.{{ flow.id }} before the next scheduled run."
}