CreateClusterAndSubmitSteps

id: aws_emr_create_cluster
namespace: company.team

tasks:
  - id: create_cluster
    type: io.kestra.plugin.aws.emr.CreateClusterAndSubmitSteps
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"
    region: eu-west-3
    clusterName: "Spark job cluster"
    logUri: "s3://my-bucket/test-emr-logs"
    keepJobFlowAliveWhenNoSteps: true
    applications:
        - Spark
    masterInstanceType: m5.xlarge
    slaveInstanceType: m5.xlarge
    instanceCount: 3
    ec2KeyName: my-ec2-ssh-key-pair-name
    steps:
        - name: Spark_job_test
          jar: "command-runner.jar"
          actionOnFailure: CONTINUE
          commands:
            - spark-submit s3://mybucket/health_violations.py --data_source s3://mybucket/food_establishment_data.csv --output_uri s3://mybucket/test-emr-output
    wait: true