diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e33c5033c..897482c0f 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import io import os import re import sys @@ -1022,6 +1023,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_headers["content-length"] = len(content) return 206, response_headers, content + def _handle_v4_chunk_signatures(self, body, content_length): + body_io = io.BytesIO(body) + new_body = bytearray(content_length) + pos = 0 + line = body_io.readline() + while line: + # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition + # str(hex(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n + chunk_size = int(line[: line.find(b";")].decode("utf8"), 16) + new_body[pos : pos + chunk_size] = body_io.read(chunk_size) + pos = pos + chunk_size + body_io.read(2) # skip trailing \r\n + line = body_io.readline() + return bytes(new_body) + def key_or_control_response(self, request, full_url, headers): # Key and Control are lumped in because splitting out the regex is too much of a pain :/ self.method = request.method @@ -1192,6 +1208,14 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if body is None: body = b"" + if ( + request.headers.get("x-amz-content-sha256", None) + == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + ): + body = self._handle_v4_chunk_signatures( + body, int(request.headers["x-amz-decoded-content-length"]) + ) + if method == "GET": return self._key_response_get( bucket_name, query, key_name, headers=request.headers diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 4171200aa..82e58bf66 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1125,6 +1125,45 @@ def test_multipart_upload_from_file_to_presigned_url(): os.remove("text.txt") +@mock_s3 +def test_put_chunked_with_v4_signature_in_body(): + bucket_name = "mybucket" + file_name = "file" + content = "CONTENT" + content_bytes = bytes(content, encoding="utf8") + # 'CONTENT' as received in moto, when PutObject is called in java AWS SDK v2 + chunked_body = b"7;chunk-signature=bd479c607ec05dd9d570893f74eed76a4b333dfa37ad6446f631ec47dc52e756\r\nCONTENT\r\n0;chunk-signature=d192ec4075ddfc18d2ef4da4f55a87dc762ba4417b3bd41e70c282f8bec2ece0\r\n\r\n" + + s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME) + s3.create_bucket(Bucket=bucket_name) + + model = MyModel(file_name, content) + model.save() + + boto_etag = s3.get_object(Bucket=bucket_name, Key=file_name)["ETag"] + + params = {"Bucket": bucket_name, "Key": file_name} + # We'll use manipulated presigned PUT, to mimick PUT from SDK + presigned_url = boto3.client("s3").generate_presigned_url( + "put_object", params, ExpiresIn=900 + ) + requests.put( + presigned_url, + data=chunked_body, + headers={ + "Content-Type": "application/octet-stream", + "x-amz-content-sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "x-amz-decoded-content-length": str(len(content_bytes)), + }, + ) + resp = s3.get_object(Bucket=bucket_name, Key=file_name) + body = resp["Body"].read() + assert body == content_bytes + + etag = resp["ETag"] + assert etag == boto_etag + + @mock_s3 def test_default_key_buffer_size(): # save original DEFAULT_KEY_BUFFER_SIZE environment variable content