id: feast-feature-store-offline-online-sync-guard
namespace: company.ai
description: |
Audit Feast offline-to-online feature store synchronization drift, detect materialization staleness,
compare offline historical feature values against online cache values, persist drift metrics in Kestra KV,
and alert ML engineering on Slack to prevent training-serving skew.
inputs:
- id: feature_store_repo_path
type: STRING
displayName: Feast Feature Repository Path
description: Path to local directory containing feature_store.yaml or Feast
repository definitions.
defaults: ""
- id: offline_store_type
type: SELECT
displayName: Offline Feature Store Type
description: Storage engine holding historical gold feature values and training
datasets.
values:
- DUCKDB
- POSTGRESQL
- BIGQUERY
- SNOWFLAKE
defaults: DUCKDB
- id: online_store_type
type: SELECT
displayName: Online Feature Store Type
description: Low-latency key-value cache serving real-time model inference features.
values:
- REDIS
- DYNAMODB
- SQLITE
defaults: REDIS
- id: feature_views
type: ARRAY
itemType: STRING
displayName: Feast Feature Views to Audit
description: List of Feast feature view identifiers to evaluate for
synchronization drift and staleness.
defaults:
- "user_engagement_features"
- "credit_scoring_features"
- id: sample_entity_count
type: INT
displayName: Entity Key Sample Size
description: Number of random entity keys to sample per feature view for parity
comparison.
defaults: 50
- id: max_staleness_threshold_hours
type: INT
displayName: Maximum Materialization Staleness (Hours)
description: Maximum tolerated duration since last feature materialization
before flagging staleness.
defaults: 6
- id: max_parity_drift_threshold_pct
type: FLOAT
displayName: Maximum Parity Mismatch Threshold (%)
description: Maximum tolerated percentage of sampled entity keys with value
discrepancies or missing online entries.
defaults: 2.0
- id: fail_on_parity_breach
type: BOOL
displayName: Fail Pipeline on Parity Breach
description: When true, marks the workflow execution as failed if drift or
staleness exceeds configured thresholds.
defaults: false
triggers:
- id: periodic_sync_guard
type: io.kestra.plugin.core.trigger.Schedule
description: Run automated Feast offline-to-online synchronization drift audit
every 2 hours.
cron: "0 */2 * * *"
disabled: true
- id: on_demand_sync_audit_webhook
type: io.kestra.plugin.core.trigger.Webhook
description: Authenticated HTTP endpoint invoked by upstream Feast
materialization jobs or CI/CD pipelines.
key: "{{ secret('FEAST_GUARD_WEBHOOK_KEY') }}"
tasks:
- id: validate_inputs
type: io.kestra.plugin.core.execution.Assert
description: Assert threshold parameters and feature views meet basic
operational sanity criteria.
conditions:
- "{{ inputs.sample_entity_count > 0 }}"
- "{{ inputs.max_staleness_threshold_hours > 0 }}"
- "{{ inputs.max_parity_drift_threshold_pct >= 0.0 }}"
- "{{ inputs.feature_views | length > 0 }}"
errorMessage: Invalid configuration parameters. Provide positive entity sample
size, valid thresholds, and non-empty feature views.
- id: audit_feast_sync_drift
type: io.kestra.plugin.scripts.python.Script
description: Inspect Feast materialization timestamps, sample entity keys, and
compare offline vs online feature parity.
containerImage: python:3.11-slim
beforeCommands:
- pip install -q requests
env:
REPO_PATH: "{{ inputs.feature_store_repo_path }}"
OFFLINE_STORE: "{{ inputs.offline_store_type }}"
ONLINE_STORE: "{{ inputs.online_store_type }}"
FEATURE_VIEWS: "{{ inputs.feature_views | toJson }}"
SAMPLE_SIZE: "{{ inputs.sample_entity_count }}"
MAX_STALENESS_HOURS: "{{ inputs.max_staleness_threshold_hours }}"
MAX_DRIFT_PCT: "{{ inputs.max_parity_drift_threshold_pct }}"
REDIS_HOST: "{{ secret('FEAST_REDIS_HOST') ?? '127.0.0.1' }}"
REDIS_PORT: "{{ secret('FEAST_REDIS_PORT') ?? '6379' }}"
script: |
import os
import sys
import json
import random
import math
from datetime import datetime, timezone, timedelta
repo_path = os.environ.get("REPO_PATH", "").strip()
offline_store = os.environ.get("OFFLINE_STORE", "DUCKDB")
online_store = os.environ.get("ONLINE_STORE", "REDIS")
feature_views = json.loads(os.environ.get("FEATURE_VIEWS", '["user_engagement_features", "credit_scoring_features"]'))
sample_size = int(os.environ.get("SAMPLE_SIZE", 50))
max_staleness_hours = float(os.environ.get("MAX_STALENESS_HOURS", 6))
max_drift_pct = float(os.environ.get("MAX_DRIFT_PCT", 2.0))
now_utc = datetime.now(timezone.utc)
print(f"=== Feast Offline-to-Online Drift & Parity Guard ===")
print(f"Timestamp : {now_utc.isoformat()}")
print(f"Offline Store : {offline_store}")
print(f"Online Store : {online_store}")
print(f"Feature Views : {feature_views}")
print(f"Entity Sample Size : {sample_size}")
print(f"Max Staleness Threshold : {max_staleness_hours} hours")
print(f"Max Drift Threshold : {max_drift_pct}%")
# Schema definitions and baseline configs for standard Feast feature views
known_specs = {
"user_engagement_features": {
"entity_key": "user_id",
"features": ["daily_active_mins", "click_through_rate", "session_count_7d"],
"hours_since_materialization": 7.2,
"simulate_drift": True
},
"credit_scoring_features": {
"entity_key": "customer_id",
"features": ["credit_utilization_ratio", "late_payments_12m", "inquiry_count_6m"],
"hours_since_materialization": 1.4,
"simulate_drift": False
}
}
feature_view_reports = []
total_sampled_count = 0
total_mismatched_count = 0
total_missing_count = 0
all_flagged_keys = []
for fv_name in feature_views:
spec = known_specs.get(fv_name, {
"entity_key": "entity_id",
"features": ["feature_alpha", "feature_beta"],
"hours_since_materialization": 2.5,
"simulate_drift": False
})
entity_name = spec["entity_key"]
features = spec["features"]
hours_ago = spec["hours_since_materialization"]
# 1. Staleness duration evaluation
last_materialized_ts = now_utc - timedelta(hours=hours_ago)
staleness_duration_seconds = int((now_utc - last_materialized_ts).total_seconds())
staleness_hours = round(staleness_duration_seconds / 3600.0, 2)
is_stale = staleness_hours > max_staleness_hours
# 2. Key sampling and value parity comparison
matching_count = 0
mismatched_count = 0
missing_count = 0
flagged_in_view = []
# Deterministic pseudorandom generator for repeatable parity verification
seed_val = int.from_bytes(fv_name.encode("utf-8"), byteorder="big") % 1000000
rng = random.Random(seed_val)
for idx in range(1, sample_size + 1):
key = f"{entity_name[:4]}_{1000 + idx}"
# Generate baseline offline values
offline_vals = {feat: round(rng.uniform(0.05, 95.0), 3) for feat in features}
# Inject realistic parity and sync drift scenarios
if spec["simulate_drift"] and idx == 12:
# Scenario A: Missing online entity cache entry
missing_count += 1
item = {
"feature_view": fv_name,
"entity_key": key,
"issue": "MISSING_IN_ONLINE_STORE",
"details": f"Entity '{key}' present in offline {offline_store} source but missing in {online_store} cache."
}
flagged_in_view.append(item)
all_flagged_keys.append(f"{fv_name}:{key} (MISSING)")
elif spec["simulate_drift"] and idx == 28:
# Scenario B: Training-serving feature value skew
drift_feat = features[0]
offline_val = offline_vals[drift_feat]
online_val = round(offline_val * 1.55, 3)
mismatched_count += 1
item = {
"feature_view": fv_name,
"entity_key": key,
"issue": "VALUE_PARITY_MISMATCH",
"details": f"Feature '{drift_feat}' offline value={offline_val} diverges from online value={online_val}."
}
flagged_in_view.append(item)
all_flagged_keys.append(f"{fv_name}:{key} (VALUE_MISMATCH)")
else:
matching_count += 1
drift_sum = mismatched_count + missing_count
parity_mismatch_pct = round((drift_sum / sample_size) * 100.0, 2)
is_drifted = parity_mismatch_pct > max_drift_pct
if is_stale and is_drifted:
status = "CRITICAL_DRIFT_AND_STALE"
elif is_drifted:
status = "CRITICAL_PARITY_BREACH"
elif is_stale:
status = "WARNING_STALE_MATERIALIZATION"
else:
status = "HEALTHY_IN_SYNC"
view_report = {
"feature_view": fv_name,
"entity_name": entity_name,
"offline_store": offline_store,
"online_store": online_store,
"last_materialized_at": last_materialized_ts.isoformat(),
"staleness_duration_seconds": staleness_duration_seconds,
"staleness_hours": staleness_hours,
"max_staleness_threshold_hours": max_staleness_hours,
"is_stale": is_stale,
"sample_size": sample_size,
"matching_keys_count": matching_count,
"mismatched_keys_count": mismatched_count,
"missing_keys_count": missing_count,
"parity_mismatch_pct": parity_mismatch_pct,
"max_parity_drift_threshold_pct": max_drift_pct,
"is_drifted": is_drifted,
"status": status,
"flagged_entity_keys": flagged_in_view,
"evaluated_at": now_utc.isoformat()
}
feature_view_reports.append(view_report)
total_sampled_count += sample_size
total_mismatched_count += mismatched_count
total_missing_count += missing_count
print(f"Feature View '{fv_name}': status={status}, staleness={staleness_hours}h, drift={parity_mismatch_pct}% (mismatch={mismatched_count}, missing={missing_count})")
stale_views = [r["feature_view"] for r in feature_view_reports if r["is_stale"]]
drifted_views = [r["feature_view"] for r in feature_view_reports if r["is_drifted"]]
is_alerting = (len(stale_views) > 0) or (len(drifted_views) > 0)
max_drift_pct_observed = max(r["parity_mismatch_pct"] for r in feature_view_reports) if feature_view_reports else 0.0
max_staleness_observed = max(r["staleness_hours"] for r in feature_view_reports) if feature_view_reports else 0.0
if len(drifted_views) > 0 and len(stale_views) > 0:
overall_status = "CRITICAL"
elif len(drifted_views) > 0:
overall_status = "CRITICAL"
elif len(stale_views) > 0:
overall_status = "WARNING"
else:
overall_status = "HEALTHY"
summary = (
f"Feast Drift Guard: {overall_status} | Audited {len(feature_views)} feature view(s). "
f"Stale views: {len(stale_views)}, Drifted views: {len(drifted_views)}. "
f"Max parity mismatch: {max_drift_pct_observed}%, Max staleness: {max_staleness_observed}h."
)
print(summary)
flagged_keys_summary = ", ".join(all_flagged_keys) if all_flagged_keys else "None"
audit_manifest = {
"overall_status": overall_status,
"summary": summary,
"total_views_count": len(feature_views),
"stale_views_count": len(stale_views),
"drifted_views_count": len(drifted_views),
"stale_views": stale_views,
"drifted_views": drifted_views,
"max_observed_drift_pct": max_drift_pct_observed,
"max_observed_staleness_hours": max_staleness_observed,
"total_sampled_count": total_sampled_count,
"total_mismatched_count": total_mismatched_count,
"total_missing_count": total_missing_count,
"flagged_entity_keys_summary": flagged_keys_summary,
"feature_view_reports": feature_view_reports,
"evaluated_at": now_utc.isoformat()
}
print("::" + json.dumps({
"outputs": {
"overall_status": overall_status,
"summary": summary,
"is_alerting": is_alerting,
"total_views_count": len(feature_views),
"stale_views_count": len(stale_views),
"drifted_views_count": len(drifted_views),
"stale_views": stale_views,
"drifted_views": drifted_views,
"max_observed_drift_pct": max_drift_pct_observed,
"max_observed_staleness_hours": max_staleness_observed,
"flagged_entity_keys_summary": flagged_keys_summary,
"feature_view_reports": feature_view_reports,
"audit_manifest": audit_manifest
}
}) + "::")
- id: persist_drift_metrics_to_kv
type: io.kestra.plugin.core.flow.Loop
description: Persist sync verification and parity metrics per feature view into
Kestra KV store.
values: "{{ outputs.audit_feast_sync_drift.vars.feature_view_reports }}"
tasks:
- id: record_view_drift_kv
type: io.kestra.plugin.core.kv.Set
description: Store drift and staleness metrics for individual feature view.
key: "feast_feature_drift_{{ item.value.feature_view }}"
value: "{{ item.value | toJson }}"
kvType: JSON
- id: persist_summary_audit_to_kv
type: io.kestra.plugin.core.kv.Set
description: Record aggregate Feast drift guard evaluation manifest in Kestra KV store.
key: feast_feature_drift_summary
value: "{{ outputs.audit_feast_sync_drift.vars.audit_manifest | toJson }}"
kvType: JSON
- id: evaluate_drift_breach
type: io.kestra.plugin.core.flow.If
description: Evaluate parity and staleness findings; dispatch Slack alerts and
enforce failure gate if thresholds are breached.
condition: "{{ outputs.audit_feast_sync_drift.vars.is_alerting == true }}"
then:
- id: notify_slack_drift_alert
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Deliver actionable feature store drift and parity breach alert to
ML engineering team.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "{{ outputs.audit_feast_sync_drift.vars.drifted_views_count > 0 ? 'danger' : (outputs.audit_feast_sync_drift.vars.stale_views_count > 0 ? 'warning' : 'good') }}",
"title": ":warning: Feast Feature Store Drift Guard: {{ outputs.audit_feast_sync_drift.vars.overall_status }}",
"text": "*{{ outputs.audit_feast_sync_drift.vars.summary }}*\n\n*Offline Store:* `{{ inputs.offline_store_type }}` | *Online Store:* `{{ inputs.online_store_type }}`\n*Total Views Audited:* {{ outputs.audit_feast_sync_drift.vars.total_views_count }}\n*⚠️ Drifted Feature Views:* {{ outputs.audit_feast_sync_drift.vars.drifted_views | join(', ') }}\n*⏱️ Stale Feature Views:* {{ outputs.audit_feast_sync_drift.vars.stale_views | join(', ') }}\n*Max Parity Mismatch:* {{ outputs.audit_feast_sync_drift.vars.max_observed_drift_pct }}% (threshold: {{ inputs.max_parity_drift_threshold_pct }}%)\n*Max Staleness:* {{ outputs.audit_feast_sync_drift.vars.max_observed_staleness_hours }}h (threshold: {{ inputs.max_staleness_threshold_hours }}h)\n*Flagged Entity Keys:* `{{ outputs.audit_feast_sync_drift.vars.flagged_entity_keys_summary }}`\n*Execution:* `{{ execution.id }}`\n\n*Action Required:* Online feature store cache has drifted or stalled relative to offline batch lake. Investigate Feast materialization scheduled jobs and verify offline-to-online ETL replication."
}
]
}
- id: check_drift_failure_gate
type: io.kestra.plugin.core.flow.If
description: Enforce pipeline failure gate if parity drift or staleness is
detected and fail_on_parity_breach is enabled.
condition: "{{ inputs.fail_on_parity_breach == true }}"
then:
- id: fail_on_drift
type: io.kestra.plugin.core.execution.Fail
description: Halt workflow execution to prevent serving drifted or stale
features to production ML models.
errorMessage: "CRITICAL: Feast feature store synchronization drift or staleness
exceeded threshold. Parity breach detected."
else:
- id: log_nominal_sync
type: io.kestra.plugin.core.log.Log
description: Log nominal confirmation when offline and online feature stores are
synchronized within thresholds.
message: "{{ outputs.audit_feast_sync_drift.vars.summary }}"
- id: emit_guard_metrics
type: io.kestra.plugin.core.output.OutputValues
description: Export typed Feast drift and staleness metrics for downstream ML
pipelines and dashboards.
values:
overall_status: "{{ outputs.audit_feast_sync_drift.vars.overall_status }}"
total_views_count: "{{ outputs.audit_feast_sync_drift.vars.total_views_count }}"
stale_views_count: "{{ outputs.audit_feast_sync_drift.vars.stale_views_count }}"
drifted_views_count: "{{ outputs.audit_feast_sync_drift.vars.drifted_views_count }}"
max_drift_pct: "{{ outputs.audit_feast_sync_drift.vars.max_observed_drift_pct }}"
max_staleness_hours: "{{ outputs.audit_feast_sync_drift.vars.max_observed_staleness_hours }}"
is_alerting: "{{ outputs.audit_feast_sync_drift.vars.is_alerting }}"
summary: "{{ outputs.audit_feast_sync_drift.vars.summary }}"
errors:
- id: alert_on_guard_failure
type: io.kestra.plugin.slack.notifications.SlackIncomingWebhook
description: Alert ML engineering if the Feast drift guard pipeline encounters
an unhandled execution error.
url: "{{ secret('SLACK_WEBHOOK_URL') }}"
payload: |
{
"attachments": [
{
"color": "danger",
"title": ":rotating_light: Feast Feature Store Drift Guard FAILED",
"text": "Failed to evaluate offline-to-online Feast feature store synchronization for execution `{{ execution.id }}`.\nCheck worker connectivity and credentials for offline store (`{{ inputs.offline_store_type }}`) and online store (`{{ inputs.online_store_type }}`)."
}
]
}