use request body in complete upload, improve exception handling

moto/s3/exceptions.py

@@ -38,3 +38,36 @@ class MissingBucket(BucketError):
             "NoSuchBucket",
             "The specified bucket does not exist",
             *args, **kwargs)
+
+
+class InvalidPartOrder(S3ClientError):
+    code = 400
+
+    def __init__(self, *args, **kwargs):
+        super(InvalidPartOrder, self).__init__(
+            "InvalidPartOrder",
+            ("The list of parts was not in ascending order. The parts "
+             "list must be specified in order by part number."),
+            *args, **kwargs)
+
+
+class InvalidPart(S3ClientError):
+    code = 400
+
+    def __init__(self, *args, **kwargs):
+        super(InvalidPart, self).__init__(
+            "InvalidPart",
+            ("One or more of the specified parts could not be found. "
+             "The part might not have been uploaded, or the specified "
+             "entity tag might not have matched the part's entity tag."),
+            *args, **kwargs)
+
+
+class EntityTooSmall(S3ClientError):
+    code = 400
+
+    def __init__(self, *args, **kwargs):
+        super(EntityTooSmall, self).__init__(
+            "EntityTooSmall",
+            "Your proposed upload is smaller than the minimum allowed object size.",
+            *args, **kwargs)

moto/s3/models.py

@@ -11,7 +11,7 @@ import six
 from bisect import insort
 from moto.core import BaseBackend
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
-from .exceptions import BucketAlreadyExists, MissingBucket
+from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall
 from .utils import clean_key_name, _VersionedKeyStore
 
 UPLOAD_ID_BYTES = 43
@@ -123,23 +123,28 @@ class FakeMultipart(object):
         rand_b64 = base64.b64encode(os.urandom(UPLOAD_ID_BYTES))
         self.id = rand_b64.decode('utf-8').replace('=', '').replace('+', '')
 
-    def complete(self):
+    def complete(self, body):
         decode_hex = codecs.getdecoder("hex_codec")
         total = bytearray()
         md5s = bytearray()
 
         last = None
-        for index, part in enumerate(self.list_parts(), start=1):
+        count = 0
+        for pn, etag in body:
+            part = self.parts.get(pn)
+            if part is None or part.etag != etag:
+                raise InvalidPart()
             if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE:
-                return None, None
+                raise EntityTooSmall()
             part_etag = part.etag.replace('"', '')
             md5s.extend(decode_hex(part_etag)[0])
             total.extend(part.value)
             last = part
+            count += 1
 
         etag = hashlib.md5()
         etag.update(bytes(md5s))
-        return total, "{0}-{1}".format(etag.hexdigest(), index)
+        return total, "{0}-{1}".format(etag.hexdigest(), count)
 
     def set_part(self, part_id, value):
         if part_id < 1:
@@ -276,10 +281,10 @@ class S3Backend(BaseBackend):
 
         return new_multipart
 
-    def complete_multipart(self, bucket_name, multipart_id):
+    def complete_multipart(self, bucket_name, multipart_id, body):
         bucket = self.get_bucket(bucket_name)
         multipart = bucket.multiparts[multipart_id]
-        value, etag = multipart.complete()
+        value, etag = multipart.complete(body)
         if value is None:
             return
         del bucket.multiparts[multipart_id]

moto/s3/responses.py

@@ -7,7 +7,7 @@ from six.moves.urllib.parse import parse_qs, urlparse
 
 from moto.core.responses import _TemplateEnvironmentMixin
 
-from .exceptions import BucketAlreadyExists, S3ClientError
+from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder
 from .models import s3_backend
 from .utils import bucket_name_from_url, metadata_from_headers
 from xml.dom import minidom
@@ -351,6 +351,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
         template = self.response_template(S3_DELETE_OBJECT_SUCCESS)
         return 204, headers, template.render(bucket=removed_key)
 
+    def _complete_multipart_body(self, body):
+        ps = minidom.parseString(body).getElementsByTagName('Part')
+        prev = 0
+        for p in ps:
+            pn = int(p.getElementsByTagName('PartNumber')[0].firstChild.wholeText)
+            if pn <= prev:
+                raise InvalidPartOrder()
+            yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText)
+
     def _key_response_post(self, request, body, parsed_url, bucket_name, query, key_name, headers):
         if body == b'' and parsed_url.query == 'uploads':
             metadata = metadata_from_headers(request.headers)
@@ -365,18 +374,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
             return 200, headers, response
 
         if 'uploadId' in query:
+            body = self._complete_multipart_body(body)
             upload_id = query['uploadId'][0]
-            key = self.backend.complete_multipart(bucket_name, upload_id)
+            key = self.backend.complete_multipart(bucket_name, upload_id, body)
-
+            template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
-            if key is not None:
+            return template.render(
-                template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
+                bucket_name=bucket_name,
-                return template.render(
+                key_name=key.name,
-                    bucket_name=bucket_name,
+                etag=key.etag,
-                    key_name=key.name,
+            )
-                    etag=key.etag,
-                )
-            template = self.response_template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
-            return 400, headers, template.render()
         elif parsed_url.query == 'restore':
             es = minidom.parseString(body).getElementsByTagName('Days')
             days = es[0].childNodes[0].wholeText
@@ -588,14 +594,6 @@ S3_MULTIPART_COMPLETE_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
 </CompleteMultipartUploadResult>
 """
 
-S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
-<Error>
-  <Code>EntityTooSmall</Code>
-  <Message>Your proposed upload is smaller than the minimum allowed object size.</Message>
-  <RequestId>asdfasdfsdafds</RequestId>
-  <HostId>sdfgdsfgdsfgdfsdsfgdfs</HostId>
-</Error>"""
-
 S3_ALL_MULTIPARTS = """<?xml version="1.0" encoding="UTF-8"?>
 <ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
   <Bucket>{{ bucket_name }}</Bucket>