Upload | Kestra

Upload

Upload files to S3

yaml
type: io.kestra.plugin.aws.s3.Upload

Examples

yaml
id: aws_s3_upload
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

  - id: myfile
    type: FILE

tasks:
  - id: upload
    type: io.kestra.plugin.aws.s3.Upload
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"
    from: "{{ inputs.myfile }}"
    bucket: "{{ inputs.bucket }}"
    key: "path/to/file"

yaml
id: upload_file_to_s3
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

  - id: file_url
    type: STRING
    defaults: https://wri-dataportal-prod.s3.amazonaws.com/manual/global_power_plant_database_v_1_3.zip

tasks:
  - id: download_file
    type: io.kestra.plugin.core.http.Download
    uri: "{{ inputs.file_url }}"

  - id: upload_to_s3
    type: io.kestra.plugin.aws.s3.Upload
    from: "{{ outputs.download_file.uri }}"
    key: powerplant/global_power_plant_database.zip
    bucket: "{{ inputs.bucket }}"
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"

yaml
id: upload_multiple_files_from_json_array
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

tasks:
  - id: download_file1
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/global_power_plant_database_v_1_3.zip"

  - id: download_file2
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/enhancing-adaptation-ambition-supplementary-materials.zip"

  - id: upload_files_to_s3
    type: io.kestra.plugin.aws.s3.Upload
    from: |
      [
        "{{ outputs.download_file1.uri }}",
        "{{ outputs.download_file2.uri }}"
      ]
    key: "path/to/files"
    bucket: "{{ inputs.bucket }}"
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"

yaml
id: upload_multiple_files_to_s3
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

tasks:
  - id: download_file1
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/global_power_plant_database_v_1_3.zip"

  - id: download_file2
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/enhancing-adaptation-ambition-supplementary-materials.zip"

  - id: upload_multiple_to_s3
    type: io.kestra.plugin.aws.s3.Upload
    from:
      - "{{ outputs.download_file1.uri }}"
      - "{{ outputs.download_file2.uri }}"
    key: "path/to/files"
    bucket: "{{ inputs.bucket }}"
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"

yaml
id: upload_multiple_files_from_json_map
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

tasks:
  - id: download_file1
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/global_power_plant_database_v_1_3.zip"

  - id: download_file2
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/enhancing-adaptation-ambition-supplementary-materials.zip"

  - id: upload_files_to_s3
    type: io.kestra.plugin.aws.s3.Upload
    from: |
      [
        "first_key": "{{ outputs.download_file1.uri }}",
        "second_key": "{{ outputs.download_file2.uri }}"
      ]
    key: "path/to/files"
    bucket: "{{ inputs.bucket }}"
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"

yaml
id: upload_multiple_files_to_s3_from_map
namespace: company.team

inputs:
  - id: bucket
    type: STRING
    defaults: my-bucket

tasks:
  - id: download_file1
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/global_power_plant_database_v_1_3.zip"

  - id: download_file2
    type: io.kestra.plugin.core.http.Download
    uri: "https://wri-dataportal-prod.s3.amazonaws.com/manual/enhancing-adaptation-ambition-supplementary-materials.zip"

  - id: upload_multiple_to_s3
    type: io.kestra.plugin.aws.s3.Upload
    from:
      firstKey: "{{ outputs.download_file1.uri }}"
      secondKey: "{{ outputs.download_file2.uri }}"
    key: "path/to/files"
    bucket: "{{ inputs.bucket }}"
    region: "{{ secret('AWS_DEFAULT_REGION') }}"
    accessKeyId: "{{ secret('AWS_ACCESS_KEY_ID') }}"
    secretKeyId: "{{ secret('AWS_SECRET_KEY_ID') }}"

Properties

bucket *string

from *arraystringobject

key *string

accessKeyId string

acl string

bucketKeyEnabled booleanstring

cacheControl string

checksum string

checksumAlgorithm string

Possible Values

CRC32CRC32_CSHA1SHA256CRC64_NVMEUNKNOWN_TO_SDK_VERSION

compatibilityMode booleanstring

Defaultfalse

contentDisposition string

contentEncoding string

contentLanguage string

contentLength integerstring

contentType string

endpointOverride string

expectedBucketOwner string

expires string

forcePathStyle booleanstring

metadata object

SubTypestring

objectLockLegalHoldStatus string

Possible Values

ONOFFUNKNOWN_TO_SDK_VERSION

objectLockMode string

Possible Values

GOVERNANCECOMPLIANCEUNKNOWN_TO_SDK_VERSION

objectLockRetainUntilDate string

region string

requestPayer string

secretKeyId string

serverSideEncryption string

Possible Values

AES256AWS_FSXAWS_KMSAWS_KMS_DSSEUNKNOWN_TO_SDK_VERSION

sessionToken string

storageClass string

Possible Values

STANDARDREDUCED_REDUNDANCYSTANDARD_IAONEZONE_IAINTELLIGENT_TIERINGGLACIERDEEP_ARCHIVEOUTPOSTSGLACIER_IRSNOWEXPRESS_ONEZONEFSX_OPENZFSFSX_ONTAPUNKNOWN_TO_SDK_VERSION

stsEndpointOverride string

stsRoleArn string

stsRoleExternalId string

stsRoleSessionDuration string

DefaultPT15M

stsRoleSessionName string

tagging object

SubTypestring

Outputs

bucket string

files object

Definitions

io.kestra.plugin.aws.s3.models.FileInfo

contentLengthinteger

contentTypestring

eTagstring

metadataobject

SubTypestring

uristring

Formaturi

versionIdstring

key string

versionId string

Metrics

file.count counter

Unitfiles

file.size counter

Unitbytes

Query data from Druid and load it to Redshift

Extract data from an API using Python, then load it to Postgres and S3

Register S3 JSON files as governed datasets and curate them with DuckDB

Plugin Weaviate for Kestra

Connect and query Apache Pinot databases using Kestra's JDBC plugin.

Plugin DataHub for Kestra

Upload

Upload files to S3

Create automations with AWS S3 Upload

Query data from Druid and load it to Redshift

Extract data from an API using Python, then load it to Postgres and S3

Register S3 JSON files as governed datasets and curate them with DuckDB

More Plugins in this Category

2.1.5

Upload Upload files to S3

Create automations with AWS S3 Upload

Query data from Druid and load it to Redshift

Extract data from an API using Python, then load it to Postgres and S3

Register S3 JSON files as governed datasets and curate them with DuckDB

More Plugins in this Category

2.1.5

Upload

Upload files to S3