Added handling for V4 signatures in PutObject body (#4201)

Co-authored-by: Łukasz Nowak <lukasz.nowak@idemia.com>

moto/s3/responses.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import io
 import os
 import re
 import sys
@@ -1022,6 +1023,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         response_headers["content-length"] = len(content)
         return 206, response_headers, content
 
+    def _handle_v4_chunk_signatures(self, body, content_length):
+        body_io = io.BytesIO(body)
+        new_body = bytearray(content_length)
+        pos = 0
+        line = body_io.readline()
+        while line:
+            # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#sigv4-chunked-body-definition
+            # str(hex(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n
+            chunk_size = int(line[: line.find(b";")].decode("utf8"), 16)
+            new_body[pos : pos + chunk_size] = body_io.read(chunk_size)
+            pos = pos + chunk_size
+            body_io.read(2)  # skip trailing \r\n
+            line = body_io.readline()
+        return bytes(new_body)
+
     def key_or_control_response(self, request, full_url, headers):
         # Key and Control are lumped in because splitting out the regex is too much of a pain :/
         self.method = request.method
@@ -1192,6 +1208,14 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         if body is None:
             body = b""
 
+        if (
+            request.headers.get("x-amz-content-sha256", None)
+            == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
+        ):
+            body = self._handle_v4_chunk_signatures(
+                body, int(request.headers["x-amz-decoded-content-length"])
+            )
+
         if method == "GET":
             return self._key_response_get(
                 bucket_name, query, key_name, headers=request.headers

tests/test_s3/test_s3.py

@@ -1125,6 +1125,45 @@ def test_multipart_upload_from_file_to_presigned_url():
     os.remove("text.txt")
 
 
+@mock_s3
+def test_put_chunked_with_v4_signature_in_body():
+    bucket_name = "mybucket"
+    file_name = "file"
+    content = "CONTENT"
+    content_bytes = bytes(content, encoding="utf8")
+    # 'CONTENT' as received in moto, when PutObject is called in java AWS SDK v2
+    chunked_body = b"7;chunk-signature=bd479c607ec05dd9d570893f74eed76a4b333dfa37ad6446f631ec47dc52e756\r\nCONTENT\r\n0;chunk-signature=d192ec4075ddfc18d2ef4da4f55a87dc762ba4417b3bd41e70c282f8bec2ece0\r\n\r\n"
+
+    s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
+    s3.create_bucket(Bucket=bucket_name)
+
+    model = MyModel(file_name, content)
+    model.save()
+
+    boto_etag = s3.get_object(Bucket=bucket_name, Key=file_name)["ETag"]
+
+    params = {"Bucket": bucket_name, "Key": file_name}
+    # We'll use manipulated presigned PUT, to mimick PUT from SDK
+    presigned_url = boto3.client("s3").generate_presigned_url(
+        "put_object", params, ExpiresIn=900
+    )
+    requests.put(
+        presigned_url,
+        data=chunked_body,
+        headers={
+            "Content-Type": "application/octet-stream",
+            "x-amz-content-sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
+            "x-amz-decoded-content-length": str(len(content_bytes)),
+        },
+    )
+    resp = s3.get_object(Bucket=bucket_name, Key=file_name)
+    body = resp["Body"].read()
+    assert body == content_bytes
+
+    etag = resp["ETag"]
+    assert etag == boto_etag
+
+
 @mock_s3
 def test_default_key_buffer_size():
     # save original DEFAULT_KEY_BUFFER_SIZE environment variable content