Destinations
S3
Concepts
Connectors
- Sources
- Destinations
Tables
Destinations
S3
Artie will write delta files in Parquet format to an S3 bucket.
Settings
- Bucket (required)
- Folder name (optional)
- Service account (required)
Bucket Structure
Artie will write all the delta files into an S3 bucket. The bucket structure will be as follows:
-- tableName will be the fully qualified table name (db.schema.tableName)
/{{bucketName}}/{{folderName}}/{{tableName}}/{{YYYY-MM-DD}}/{{unixTimestampMs}}_{{randomString(4)}}.parquet.gz
-- example
/artie/foo/db.schema.tableName/2023-08-06/1654320000000_abc1.parquet.gz
provider "aws" {
region = "us-east-1" # Your AWS Region
}
resource "aws_iam_user" "artie_transfer" {
name = "artie-transfer" # Your service account name
}
resource "aws_iam_policy" "s3_read_write_policy" {
name = "s3-read-write-policy"
description = "Ability to list, read and write to the bucket"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Effect = "Allow"
Action = [
"s3:ListBucket"
]
Resource = "arn:aws:s3:::your-bucket-name"
},
{
Effect = "Allow"
Action = [
"s3:GetObject",
"s3:PutObject"
]
Resource = "arn:aws:s3:::your-bucket-name/*"
}
]
})
}
resource "aws_iam_user_policy_attachment" "user_policy_attachment" {
user = aws_iam_user.artie_transfer.name
policy_arn = aws_iam_policy.s3_read_write_policy.arn
}
resource "aws_iam_access_key" "artie_transfer_key" {
user = aws_iam_user.artie_transfer.name
}
output "access_key_id" {
value = aws_iam_access_key.artie_transfer_key.id
sensitive = false
}
output "secret_access_key" {
value = aws_iam_access_key.artie_transfer_key.secret
sensitive = true
}
On this page