Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ erl_crash.dump
/tmp
.elixir_ls
*~
.*-credentials.json
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,24 @@ Multipart uploads
```
See `ExAws.S3.upload/4` for options

Multipart uploads with SSE-C (server-side encryption with a customer-provided key).
The same `:encryption` opt is applied to the initiate, each part upload, and the
complete request.
```elixir
key = :crypto.strong_rand_bytes(32)

encryption = [
customer_algorithm: "AES256",
customer_key: Base.encode64(key),
customer_key_md5: Base.encode64(:crypto.hash(:md5, key))
]

"path/to/big/file"
|> S3.Upload.stream_file
|> S3.upload("my-bucket", "path/on/s3", encryption: encryption)
|> ExAws.request #=> {:ok, :done}
```

Download large file to disk
```elixir
S3.download_file("my-bucket", "path/on/s3", "path/to/dest/file")
Expand Down
18 changes: 16 additions & 2 deletions config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,23 @@ if config_env() == :test and System.get_env("MIX_TEST_EX_AWS_MINIO") do
access_key_id: System.get_env("MINIO_ROOT_USER") || "minio",
secret_access_key: System.get_env("MINIO_ROOT_PASSWORD") || "miniosecret",
s3: [
scheme: "http://",
scheme: System.get_env("MINIO_SCHEME") || "https://",
host: System.get_env("MINIO_HOSTNAME") || "localhost",
port: System.get_env("MINIO_PORT") || 9000,
region: "us-east-1"
region: System.get_env("MINIO_REGION") || "us-east-1"
]
end

if config_env() == :test and System.get_env("MIX_TEST_EX_AWS_HETZNER") do
config :ex_aws,
access_key_id: System.get_env("S3_ROOT_USER"),
secret_access_key: System.get_env("S3_ROOT_PASSWORD"),
sse_customer_key: System.get_env("SSE_CUSTOMER_KEY"),
bucket: System.get_env("S3_BUCKET"),
s3: [
scheme: System.get_env("S3_SCHEME") || "https://",
host: System.get_env("S3_HOSTNAME"),
port: System.get_env("S3_PORT") || 443,
region: System.get_env("S3_REGION") || "fsn1"
]
end
135 changes: 135 additions & 0 deletions justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# Elixir/Docker image used for in-container tasks.

elixir_image := env("elixir_image", "elixir:1.18.4-otp-26-alpine")

# MinIO image and container name for the S3 integration tests.
# Override to match CI's pinned tag, e.g.
# minio_image=quay.io/minio/minio:RELEASE.2025-07-23T15-54-02Z just in-docker-minio-test

minio_image := env("minio_image", "minio/minio:latest")
minio_container := "ex-aws-s3-minio"

# show all recipes
@recipes:
just --list --justfile {{justfile()}}

# Run the test suite inside a Docker container.
in-docker-test:
docker run --rm \
-v "{{ justfile_directory() }}:/app" \
-w /app \
"{{ elixir_image }}" \
sh -c "mix local.hex --force && mix local.rebar --force && mix deps.get && mix test"

# Run the MinIO integration tests against a real S3 endpoint in Docker.
# Starts a MinIO container with TLS (self-signed cert), waits for it to be
# ready, then runs the `:minio`-tagged tests (via the `test.minio` alias)
# in the Elixir container.

# Run the tests against a docker minio container
in-docker-minio-test:
#!/usr/bin/env bash
set -euo pipefail

# Generate a self-signed TLS certificate for MinIO
certs_dir="$(mktemp -d)"
echo "Certs dir: ${certs_dir}"
trap 'docker rm -f {{ minio_container }} >/dev/null 2>&1 || true; rm -rf "$certs_dir"' EXIT

openssl req -x509 -newkey rsa:2048 -nodes \
-keyout "$certs_dir/private.key" \
-out "$certs_dir/public.crt" \
-days 1 \
-subj "/CN=localhost" \
-addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \
2>/dev/null

docker rm -f {{ minio_container }} >/dev/null 2>&1 || true
docker run -d \
--name {{ minio_container }} \
-p 9000:9000 \
-v "$certs_dir:/root/.minio/certs:ro" \
-e MINIO_ROOT_USER=minio \
-e MINIO_ROOT_PASSWORD=miniosecret \
-e MINIO_NOTIFY_WEBHOOK_ENABLE_testhook=on \
-e MINIO_NOTIFY_WEBHOOK_ENDPOINT_testhook=http://localhost:9999 \
{{ minio_image }} \
server /data --address ":9000"

echo "Waiting for MinIO (TLS) to be ready..."
timeout 60s bash -c 'until curl -fsk https://localhost:9000/minio/health/live >/dev/null; do echo " not ready yet..."; sleep 2; done'

docker run --rm \
--network host \
-v "{{ justfile_directory() }}:/app" \
-v "$certs_dir:/minio/certs:ro" \
-w /app \
-e MIX_TEST_EX_AWS_MINIO=true \
-e MINIO_SCHEME=https:// \
-e MINIO_HOSTNAME=localhost \
-e MINIO_PORT=9000 \
-e MINIO_CA_CERT=/minio/certs/public.crt \
"{{ elixir_image }}" \
sh -c "mix local.hex --force && mix local.rebar --force && mix deps.get && mix test.minio"

# create .sops.yaml
#creation_rules:
# - path_regex: .*.json
# key_groups:
# - age:
# - the-public-key
# credentials file: .hetzner-credentials.json
# {
# "HETZNER_HOSTNAME": "fsn1.your-objectstorage.com",
# "HETZNER_BUCKET": "my-bucket-name",
# "HETZNER_REGION": "fsn1",
# "HETZNER_USER": "",
# "HETZNER_PASSWORD": "",
# "HETZNER_SSE_CUSTOMER_ALGORITHM": "AES256",
# "HETZNER_SSE_CUSTOMER_KEY": "" # 32 char random key
# }
# encrypt it: sops encrypt --in-place .hetzner-credentials.json
# Run the tests against a real hetzner object storage
in-docker-hetzner-test CREDENTIALS=".hetzner-credentials.json":
#!/usr/bin/env bash
set -euo pipefail

docker run --rm \
--network host \
-v "{{ justfile_directory() }}:/app" \
-w /app \
-e MIX_TEST_EX_AWS_HETZNER=true \
-e S3_ROOT_USER=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_USER) \
-e S3_ROOT_PASSWORD=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_PASSWORD) \
-e S3_BUCKET=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_BUCKET) \
-e S3_SCHEME="https://" \
-e S3_HOSTNAME=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_HOSTNAME) \
-e S3_PORT=443 \
-e S3_REGION=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_REGION) \
-e SSE_CUSTOMER_KEY=$(sops decrypt {{ CREDENTIALS }} | jq -r .HETZNER_SSE_CUSTOMER_KEY) \
"{{ elixir_image }}" \
sh -c "mix local.hex --force && mix local.rebar --force && mix deps.get && mix test.hetzner"

# run hex format
format:
#!/usr/bin/env bash
set -euo pipefail

docker run --rm \
--network host \
-v "{{ justfile_directory() }}:/app" \
-w /app \
"{{ elixir_image }}" \
sh -c "mix local.hex --force && mix local.rebar --force && mix deps.get && mix format"

# run hex dialyzer
dialyzer:
#!/usr/bin/env bash
set -euo pipefail

docker run --rm \
--network host \
-v "{{ justfile_directory() }}:/app" \
-w /app \
"{{ elixir_image }}" \
sh -c "mix local.hex --force && mix local.rebar --force && mix deps.get && mix dialyzer"
38 changes: 28 additions & 10 deletions lib/ex_aws/s3.ex
Original file line number Diff line number Diff line change
Expand Up @@ -1320,7 +1320,8 @@ defmodule ExAws.S3 do
@type download_file_opts :: [
max_concurrency: pos_integer,
chunk_size: pos_integer,
timeout: pos_integer
timeout: pos_integer,
encryption: encryption_opts
]

@doc ~S"""
Expand Down Expand Up @@ -1440,8 +1441,9 @@ defmodule ExAws.S3 do
the initial auth. Fixes an edge case uploading large files when using a strategy from `ex_aws_sts` that provides short lived tokens,
where uploads could fail if the token expires before the upload is completed. Defaults to `false`.

All other options (ex. `:content_type`) are passed through to
`ExAws.S3.initiate_multipart_upload/3`.
All other options (ex. `:content_type`, `:encryption`) are passed through to
`ExAws.S3.initiate_multipart_upload/3`, `ExAws.S3.upload_part/6`, and
`ExAws.S3.complete_multipart_upload/5`.

"""
@spec upload(
Expand Down Expand Up @@ -1806,11 +1808,12 @@ defmodule ExAws.S3 do
upload_id :: binary,
part_number :: pos_integer,
body :: binary,
opts :: [encryption_opts | {:expect, binary}]
opts :: [encryption: encryption_opts, expect: binary]
) :: ExAws.Operation.S3.t()
def upload_part(bucket, object, upload_id, part_number, body, _opts \\ []) do
def upload_part(bucket, object, upload_id, part_number, body, opts \\ []) do
params = %{"uploadId" => upload_id, "partNumber" => part_number}
request(:put, bucket, object, params: params, body: body)
headers = opts |> Map.new() |> Map.get(:encryption, %{}) |> build_encryption_headers
request(:put, bucket, object, params: params, body: body, headers: headers)
end

@type upload_part_copy_opts :: [
Expand Down Expand Up @@ -1893,14 +1896,23 @@ defmodule ExAws.S3 do
})
end

@type complete_multipart_upload_opts :: [encryption: encryption_opts]

@doc "Complete a multipart upload"
@spec complete_multipart_upload(
bucket :: binary,
object :: binary,
upload_id :: binary,
parts :: [{binary | pos_integer, binary}, ...]
) :: ExAws.Operation.S3.t()
def complete_multipart_upload(bucket, object, upload_id, parts) do
@spec complete_multipart_upload(
bucket :: binary,
object :: binary,
upload_id :: binary,
parts :: [{binary | pos_integer, binary}, ...],
opts :: complete_multipart_upload_opts
) :: ExAws.Operation.S3.t()
def complete_multipart_upload(bucket, object, upload_id, parts, opts \\ []) do
parts_xml =
parts
|> Enum.map(fn {part_number, etag} ->
Expand All @@ -1920,9 +1932,15 @@ defmodule ExAws.S3 do
["<CompleteMultipartUpload>", parts_xml, "</CompleteMultipartUpload>"]
|> IO.iodata_to_binary()

request(:post, bucket, object, [params: %{"uploadId" => upload_id}, body: body], %{
parser: &Parsers.parse_complete_multipart_upload/1
})
headers = opts |> Map.new() |> Map.get(:encryption, %{}) |> build_encryption_headers

request(
:post,
bucket,
object,
[params: %{"uploadId" => upload_id}, body: body, headers: headers],
%{parser: &Parsers.parse_complete_multipart_upload/1}
)
end

@doc "Abort a multipart upload"
Expand Down
14 changes: 9 additions & 5 deletions lib/ex_aws/s3/download.ex
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,18 @@ defmodule ExAws.S3.Download do
def get_chunk(op, %{start_byte: start_byte, end_byte: end_byte}, config) do
%{body: body} =
op.bucket
|> ExAws.S3.get_object(op.path, range: "bytes=#{start_byte}-#{end_byte}")
|> ExAws.S3.get_object(
op.path,
Keyword.merge(op.opts, range: "bytes=#{start_byte}-#{end_byte}")
)
|> ExAws.request!(config)

{start_byte, body}
end

def build_chunk_stream(op, config) do
op.bucket
|> get_file_size(op.path, config)
op
|> get_file_size(config)
|> chunk_stream(op.opts[:chunk_size] || 1024 * 1024)
end

Expand All @@ -47,8 +50,9 @@ defmodule ExAws.S3.Download do
end)
end

defp get_file_size(bucket, path, config) do
%{headers: headers} = ExAws.S3.head_object(bucket, path) |> ExAws.request!(config)
defp get_file_size(op, config) do
%{headers: headers} =
ExAws.S3.head_object(op.bucket, op.path, op.opts) |> ExAws.request!(config)

headers
|> Enum.find(fn {k, _} -> String.downcase(k) == "content-length" end)
Expand Down
3 changes: 2 additions & 1 deletion lib/ex_aws/s3/upload.ex
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ defmodule ExAws.S3.Upload do
op.bucket,
op.path,
op.upload_id,
Enum.sort_by(parts, &elem(&1, 0))
Enum.sort_by(parts, &elem(&1, 0)),
op.opts
)
|> ExAws.request(config)
end
Expand Down
3 changes: 2 additions & 1 deletion mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ defmodule ExAws.S3.Mixfile do

defp aliases do
[
"test.minio": ["cmd env MIX_TEST_EX_AWS_MINIO=true mix test --only minio"]
"test.minio": ["cmd env MIX_TEST_EX_AWS_MINIO=true mix test --only minio"],
"test.hetzner": ["cmd env MIX_TEST_EX_AWS_HETZNER=true mix test --only hetzner"]
]
end

Expand Down
Loading