id: sentinel2-cloud-masked-ndvi-geotiff
namespace: company.team
description: Create a bounded, cloud-masked Sentinel-2 NDVI GeoTIFF and verify
its read-back before packaging.
inputs:
- id: bbox
type: STRING
defaults: '[-119.80,36.70,-119.78,36.72]'
description: WGS84 [west,south,east,north], max 0.05 degrees per side, entirely
inside one tile.
- id: start_date
type: DATE
defaults: '{{ now() | dateAdd(-30, ''DAYS'') | date(''yyyy-MM-dd'') }}'
description: Start of inclusive UTC search interval.
- id: end_date
type: DATE
defaults: '{{ now() | date(''yyyy-MM-dd'') }}'
description: End of inclusive UTC search interval.
- id: max_cloud_pct
type: FLOAT
defaults: 20
description: Maximum tile cloud percentage, 0..100. AOI quality is checked separately.
- id: min_valid_fraction
type: FLOAT
defaults: 0.5
description: Minimum AOI pixel fraction classified as vegetation, bare ground or
water, >0 and <=1.
triggers:
- id: weekly
type: io.kestra.plugin.core.trigger.Schedule
cron: 0 6 * * 1
disabled: true
tasks:
- id: validate_inputs
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
description: Reject invalid AOI, dates and cloud limit before network requests.
env:
BBOX: '{{ inputs.bbox }}'
START: '{{ inputs.start_date | date(''yyyy-MM-dd'') }}'
END: '{{ inputs.end_date | date(''yyyy-MM-dd'') }}'
MAX_CLOUD: '{{ inputs.max_cloud_pct }}'
MIN_VALID: '{{ inputs.min_valid_fraction }}'
script: |
import json, math, os
from datetime import date
b=json.loads(os.environ['BBOX'])
if len(b)!=4 or any(not isinstance(x,(int,float)) or not math.isfinite(x) for x in b):
raise ValueError('bbox must contain four finite numbers')
w,s,e,n=b
if not (-180<=w<e<=180 and -80<=s<n<=80) or max(e-w,n-s)>0.05:
raise ValueError('Use a non-antimeridian AOI at most 0.05 degrees wide/high within latitude +/-80')
if date.fromisoformat(os.environ['START'])>date.fromisoformat(os.environ['END']):
raise ValueError('start_date must precede end_date')
limit=float(os.environ['MAX_CLOUD'])
if not 0<=limit<=100: raise ValueError('max_cloud_pct must be 0..100')
quality=float(os.environ["MIN_VALID"])
if not 0<quality<=1: raise ValueError("min_valid_fraction must be >0 and <=1")
print('::{"outputs":'+json.dumps({"bbox_csv":",".join(str(v) for v in b)})+'}::')
- id: search_catalog
type: io.kestra.plugin.core.http.Download
description: Search the public Element 84 Earth Search STAC catalog for at most
100 scenes.
uri: https://earth-search.aws.element84.com/v1/search?collections=sentinel-2-l2a&bbox={{
outputs.validate_inputs.vars.bbox_csv }}&datetime={{ inputs.start_date |
date('yyyy-MM-dd') }}T00:00:00Z/{{ inputs.end_date | date('yyyy-MM-dd')
}}T23:59:59Z&limit=100
- id: select_scene
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
description: Choose the least-cloudy returned tile containing the entire AOI;
record catalog truncation.
env:
BBOX: '{{ inputs.bbox }}'
START: '{{ inputs.start_date | date(''yyyy-MM-dd'') }}'
END: '{{ inputs.end_date | date(''yyyy-MM-dd'') }}'
MAX_CLOUD: '{{ inputs.max_cloud_pct }}'
inputFiles:
catalog.json: '{{ outputs.search_catalog.uri }}'
outputFiles:
- selection.json
script: |
import json, math, os
from datetime import date
b=json.loads(os.environ['BBOX'])
if len(b)!=4 or any(not isinstance(x,(int,float)) or not math.isfinite(x) for x in b):
raise ValueError('bbox must contain four finite numbers')
w,s,e,n=b
if not (-180<=w<e<=180 and -80<=s<n<=80) or max(e-w,n-s)>0.05:
raise ValueError('Use a non-antimeridian AOI at most 0.05 degrees wide/high within latitude +/-80')
if date.fromisoformat(os.environ['START'])>date.fromisoformat(os.environ['END']):
raise ValueError('start_date must precede end_date')
limit=float(os.environ['MAX_CLOUD'])
if not 0<=limit<=100: raise ValueError('max_cloud_pct must be 0..100')
x=json.load(open('catalog.json'))
if x.get('type')!='FeatureCollection': raise ValueError('Expected a STAC FeatureCollection')
candidates=[]
for f in x['features']:
p=f.get('properties',{}); a=f.get('assets',{})
if p.get('eo:cloud_cover',101)>limit or not all(k in a for k in ['red','nir','scl']): continue
# An entire AOI must fit in one tile; do not silently produce a partial raster.
fb=f.get('bbox',[])
if len(fb)!=4 or not (fb[0]<=w<e<=fb[2] and fb[1]<=s<n<=fb[3]): continue
for k in ['red','nir','scl']:
if not a[k]['href'].startswith('https://sentinel-cogs.s3.us-west-2.amazonaws.com/'):
raise ValueError('Unexpected asset host')
candidates.append(f)
candidates.sort(key=lambda f:(f['properties']['eo:cloud_cover'], f['id']))
chosen=candidates[0] if candidates else None
json.dump({'scene':chosen,'bbox':b,'candidate_count':len(candidates),'returned_count':len(x['features']),
'catalog_truncated':any(l.get('rel')=='next' for l in x.get('links',[]))},open('selection.json','w'),indent=2)
print('::{"outputs":'+json.dumps({'found':bool(chosen),'scene_id':chosen['id'] if chosen else '',
'candidate_count':len(candidates)})+'}::')
- id: scene_available
type: io.kestra.plugin.core.flow.If
condition: '{{ outputs.select_scene.vars.found }}'
then:
- id: create_ndvi
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
description: Read only AOI windows over HTTPS, apply STAC calibration and SCL
cloud masks, and write a real NDVI GeoTIFF.
env:
MIN_VALID: '{{ inputs.min_valid_fraction }}'
dependencies:
- numpy==2.2.6
- rasterio==1.4.3
inputFiles:
selection.json: '{{ outputs.select_scene.outputFiles[''selection.json''] }}'
outputFiles:
- ndvi.tif
- manifest.json
script: |
import json, os
import numpy as np
import rasterio
from rasterio.warp import transform_bounds, reproject, Resampling
from rasterio.windows import from_bounds
sel=json.load(open('selection.json')); f=sel['scene']; assets=f['assets']
if f is None: raise ValueError('No scene selected')
quality=float(os.environ['MIN_VALID'])
if not 0<quality<=1: raise ValueError('min_valid_fraction must be >0 and <=1')
env=dict(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',CPL_VSIL_CURL_ALLOWED_EXTENSIONS='.tif',GDAL_HTTP_TIMEOUT='60')
with rasterio.Env(**env):
with rasterio.open(assets['red']['href']) as red:
bounds=transform_bounds('EPSG:4326',red.crs,*sel['bbox'],densify_pts=21)
window=from_bounds(*bounds,transform=red.transform).round_offsets().round_lengths()
if window.width<1 or window.height<1 or window.width*window.height>1_000_000:
raise ValueError('Empty or oversized AOI window')
if window.col_off<0 or window.row_off<0 or window.col_off+window.width>red.width or window.row_off+window.height>red.height:
raise ValueError('AOI falls outside the selected tile grid')
r=red.read(1,window=window,masked=True); transform=red.window_transform(window)
profile=red.profile.copy(); crs=red.crs
with rasterio.open(assets['nir']['href']) as nir:
if nir.crs!=crs or nir.window_transform(window)!=transform:
raise ValueError('Red and NIR grids differ')
ir=nir.read(1,window=window,masked=True)
with rasterio.open(assets['scl']['href']) as scl:
# Read only the overlapping 20 m window, then align classes with the 10 m grid.
sw=from_bounds(*transform_bounds(crs,scl.crs,*bounds),transform=scl.transform).round_offsets().round_lengths()
source=scl.read(1,window=sw)
classes=np.zeros(r.shape,dtype='uint8')
reproject(source,classes,src_transform=scl.window_transform(sw),src_crs=scl.crs,
dst_transform=transform,dst_crs=crs,resampling=Resampling.nearest,
src_nodata=0,dst_nodata=0)
def reflectance(raw,asset):
band=asset.get('raster:bands',[{}])[0]
if 'scale' not in band or 'offset' not in band:
raise ValueError('Missing reflectance scale/offset in STAC metadata')
return raw.astype('float32')*float(band['scale'])+float(band['offset'])
rf=reflectance(r,assets['red']); nf=reflectance(ir,assets['nir']); denom=nf+rf
valid=(~np.ma.getmaskarray(r)) & (~np.ma.getmaskarray(ir)) & np.isin(classes,[4,5,6])
valid &= (denom.filled(0)>0) & (rf.filled(-1)>=0) & (nf.filled(-1)>=0)
fraction=float(valid.sum()/valid.size)
if fraction<quality:
raise ValueError(f'Only {fraction:.3f} valid pixels; need {quality:.3f}. No NDVI package published')
ndvi=np.full(r.shape,-9999,dtype='float32')
ndvi[valid]=((nf-rf)/denom).filled(-9999)[valid]
if not np.isfinite(ndvi[valid]).all() or (np.abs(ndvi[valid])>1.00001).any():
raise ValueError('NDVI outside physical bounds')
profile.update(driver='GTiff',width=r.shape[1],height=r.shape[0],transform=transform,
dtype='float32',nodata=-9999,count=1,compress='deflate')
with rasterio.open('ndvi.tif','w',**profile) as dst: dst.write(ndvi,1)
manifest={'scene_id':f['id'],'datetime':f['properties']['datetime'],'bbox':sel['bbox'],
'tile_cloud_pct':f['properties']['eo:cloud_cover'],'valid_fraction':fraction,
'valid_pixels':int(valid.sum()),'total_pixels':int(valid.size),'shape':list(r.shape),
'ndvi_mean':float(ndvi[valid].mean()),'crs':str(crs),'scl_classes':[4,5,6],
'assets':{k:assets[k]['href'] for k in ['red','nir','scl']},
'calibration':{k:assets[k]['raster:bands'][0] for k in ['red','nir']},
'candidate_count':sel['candidate_count'],'catalog_truncated':sel['catalog_truncated']}
json.dump(manifest,open('manifest.json','w'),indent=2)
print('::{"outputs":'+json.dumps({'valid_fraction':fraction,'valid_pixels':int(valid.sum())})+'}::')
- id: verify_raster
type: io.kestra.plugin.scripts.python.Script
taskRunner:
type: io.kestra.plugin.core.runner.Process
description: Reopen the artifact and verify dimensions, CRS, nodata mask, pixel
count and NDVI range before hashing.
dependencies:
- numpy==2.2.6
- rasterio==1.4.3
inputFiles:
ndvi.tif: '{{ outputs.create_ndvi.outputFiles[''ndvi.tif''] }}'
manifest.json: '{{ outputs.create_ndvi.outputFiles[''manifest.json''] }}'
outputFiles:
- SHA256SUMS
script: |
import hashlib,json
import numpy as np
import rasterio
m=json.load(open('manifest.json'))
with rasterio.open('ndvi.tif') as ds:
a=ds.read(1,masked=True)
assert list(a.shape)==m['shape']
assert a.count()==m['valid_pixels']
assert ds.nodata==-9999 and ds.crs.to_string()==m['crs']
assert np.isfinite(a.compressed()).all() and np.abs(a.compressed()).max()<=1.00001
assert abs(float(a.mean())-m['ndvi_mean'])<1e-6
with open('SHA256SUMS','w') as out:
for name in ['ndvi.tif','manifest.json']:
out.write(hashlib.sha256(open(name,'rb').read()).hexdigest()+' '+name+'\n')
print('GeoTIFF read-back, validity mask, physical bounds and manifest verified')
- id: package_verified_raster
type: io.kestra.plugin.compress.ArchiveCompress
description: Publish a ZIP only after the raster and manifest passed read-back
validation.
algorithm: ZIP
from:
ndvi.tif: '{{ outputs.create_ndvi.outputFiles[''ndvi.tif''] }}'
manifest.json: '{{ outputs.create_ndvi.outputFiles[''manifest.json''] }}'
SHA256SUMS: '{{ outputs.verify_raster.outputFiles[''SHA256SUMS''] }}'
else:
- id: no_matching_scene
type: io.kestra.plugin.core.log.Log
message: No returned tile covers the full AOI within the cloud threshold. No
raster or ZIP created.