Skip to main content

​
Required settings

  • Bucket (required)
  • Folder name (optional)
  • Checkpoint interval (optional, default: 10)
  • Service account (required)

​
Bucket structure

Artie will write all the delta files into an S3 bucket. The bucket structure will be as follows: 
-- tableName will be the fully qualified table name (db.schema.tableName)
/{{bucketName}}/{{folderName}}/{{tableName}}/{{YYYY-MM-DD}}/{{unixTimestampMs}}_{{randomString(4)}}.parquet.gz

-- example
/artie/foo/db.schema.tableName/2023-08-06/1654320000000_abc1.parquet.gz

​
Checkpoint interval

Artie periodically writes a Delta Lake checkpoint file to consolidate the transaction log. The checkpoint interval controls how many commits occur between each checkpoint. A lower value means more frequent checkpoints (faster reads, more write overhead); a higher value means fewer checkpoints (less write overhead, slower cold reads). The default is 10. 
provider "aws" {
  region = "us-east-1" # Your AWS Region
}

resource "aws_iam_user" "artie_transfer" {
  name = "artie-transfer" # Your service account name
}

resource "aws_iam_policy" "s3_read_write_policy" {
  name        = "s3-read-write-policy"
  description = "Ability to list, read and write to the bucket"

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Action = [
          "s3:ListBucket",
          "s3:GetEncryptionConfiguration"
        ]
        Resource = "arn:aws:s3:::your-bucket-name"
      },
      {
        Effect = "Allow"
        Action = [
          "s3:GetObject",
          "s3:PutObject"
        ]
        Resource = "arn:aws:s3:::your-bucket-name/*"
      },
      // If the bucket is encrypted with KMS:
      {
        Effect = "Allow",
        Action = [
          "kms:GenerateDataKey",
          "kms:Decrypt",
        ],
        Resource = [
          "arn:aws:kms:::key/your-kms-key-id",
        ]
      }
    ]
  })
}

resource "aws_iam_user_policy_attachment" "user_policy_attachment" {
  user       = aws_iam_user.artie_transfer.name
  policy_arn = aws_iam_policy.s3_read_write_policy.arn
}

resource "aws_iam_access_key" "artie_transfer_key" {
  user = aws_iam_user.artie_transfer.name
}

output "access_key_id" {
  value     = aws_iam_access_key.artie_transfer_key.id
  sensitive = false
}

output "secret_access_key" {
  value     = aws_iam_access_key.artie_transfer_key.secret
  sensitive = true
}