S3 ListParts API: use MaxParts parameter (#3658)

AUTHORS.md

@@ -58,4 +58,5 @@ Moto is written by Steve Pulec with contributions from:
 * [Craig Anderson](https://github.com/craiga)
 * [Robert Lewis](https://github.com/ralewis85)
 * [Kyle Jones](https://github.com/Kerl1310)
-* [Ariel Beck](https://github.com/arielb135)
+* [Mickaël Schoentgen](https://github.com/BoboTiG)
+* [Ariel Beck](https://github.com/arielb135)

IMPLEMENTATION_COVERAGE.md

@@ -9485,6 +9485,9 @@
 - [ ] list_objects
 - [ ] list_objects_v2
 - [ ] list_parts
+- [X] list_objects
+- [X] list_objects_v2
+- [X] list_parts
 - [X] put_bucket_accelerate_configuration
 - [X] put_bucket_acl
 - [ ] put_bucket_analytics_configuration

moto/s3/exceptions.py

@@ -237,6 +237,16 @@ class CrossLocationLoggingProhibitted(S3ClientError):
         )
 
 
+class InvalidMaxPartArgument(S3ClientError):
+    code = 400
+
+    def __init__(self, arg, min_val, max_val):
+        error = "Argument {} must be an integer between {} and {}".format(
+            arg, min_val, max_val
+        )
+        super(InvalidMaxPartArgument, self).__init__("InvalidArgument", error)
+
+
 class InvalidNotificationARN(S3ClientError):
     code = 400
 

moto/s3/models.py

@@ -369,9 +369,11 @@ class FakeMultipart(BaseModel):
             insort(self.partlist, part_id)
         return key
 
-    def list_parts(self):
+    def list_parts(self, part_number_marker, max_parts):
         for part_id in self.partlist:
-            yield self.parts[part_id]
+            part = self.parts[part_id]
+            if part_number_marker <= part.name < part_number_marker + max_parts:
+                yield part
 
 
 class FakeGrantee(BaseModel):
@@ -1712,6 +1714,20 @@ class S3Backend(BaseBackend):
             raise NoSuchUpload(upload_id=multipart_id)
         del bucket.multiparts[multipart_id]
 
+    def list_multipart(
+        self, bucket_name, multipart_id, part_number_marker=0, max_parts=1000
+    ):
+        bucket = self.get_bucket(bucket_name)
+        if multipart_id not in bucket.multiparts:
+            raise NoSuchUpload(upload_id=multipart_id)
+        return list(
+            bucket.multiparts[multipart_id].list_parts(part_number_marker, max_parts)
+        )
+
+    def is_truncated(self, bucket_name, multipart_id, next_part_number_marker):
+        bucket = self.get_bucket(bucket_name)
+        return len(bucket.multiparts[multipart_id].parts) >= next_part_number_marker
+
     def create_multipart_upload(self, bucket_name, key_name, metadata, storage_type):
         multipart = FakeMultipart(key_name, metadata)
         multipart.storage = storage_type
@@ -1728,12 +1744,6 @@ class S3Backend(BaseBackend):
             del bucket.multiparts[multipart_id]
         return multipart, value, etag
 
-    def list_multipart(self, bucket_name, multipart_id):
-        bucket = self.get_bucket(bucket_name)
-        if multipart_id not in bucket.multiparts:
-            raise NoSuchUpload(upload_id=multipart_id)
-        return list(bucket.multiparts[multipart_id].list_parts())
-
     def get_all_multiparts(self, bucket_name):
         bucket = self.get_bucket(bucket_name)
         return bucket.multiparts

moto/s3/responses.py

@@ -45,6 +45,7 @@ from .exceptions import (
     MissingBucket,
     MissingKey,
     MissingVersion,
+    InvalidMaxPartArgument,
     InvalidPartOrder,
     MalformedXML,
     MalformedACLError,
@@ -1240,7 +1241,28 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         response_headers = {}
         if query.get("uploadId"):
             upload_id = query["uploadId"][0]
-            parts = self.backend.list_multipart(bucket_name, upload_id)
+
+            # 0 <= PartNumberMarker <= 2,147,483,647
+            part_number_marker = int(query.get("part-number-marker", [0])[0])
+            if not (0 <= part_number_marker <= 2147483647):
+                raise InvalidMaxPartArgument("part-number-marker", 0, 2147483647)
+
+            # 0 <= MaxParts <= 2,147,483,647 (default is 1,000)
+            max_parts = int(query.get("max-parts", [1000])[0])
+            if not (0 <= max_parts <= 2147483647):
+                raise InvalidMaxPartArgument("max-parts", 0, 2147483647)
+
+            parts = self.backend.list_multipart(
+                bucket_name,
+                upload_id,
+                part_number_marker=part_number_marker,
+                max_parts=max_parts,
+            )
+            next_part_number_marker = parts[-1].name + 1 if parts else 0
+            is_truncated = parts and self.backend.is_truncated(
+                bucket_name, upload_id, next_part_number_marker
+            )
+
             template = self.response_template(S3_MULTIPART_LIST_RESPONSE)
             return (
                 200,
@@ -1249,8 +1271,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
                     bucket_name=bucket_name,
                     key_name=key_name,
                     upload_id=upload_id,
-                    count=len(parts),
+                    is_truncated=str(is_truncated).lower(),
+                    max_parts=max_parts,
+                    next_part_number_marker=next_part_number_marker,
                     parts=parts,
+                    part_number_marker=part_number_marker,
                 ),
             )
         version_id = query.get("versionId", [None])[0]
@@ -2358,10 +2383,10 @@ S3_MULTIPART_LIST_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
     <ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>
     <DisplayName>webfile</DisplayName>
   </Owner>
-  <PartNumberMarker>1</PartNumberMarker>
+  <PartNumberMarker>{{ part_number_marker }}</PartNumberMarker>
-  <NextPartNumberMarker>{{ count }}</NextPartNumberMarker>
+  <NextPartNumberMarker>{{ next_part_number_marker }}</NextPartNumberMarker>
-  <MaxParts>{{ count }}</MaxParts>
+  <MaxParts>{{ max_parts }}</MaxParts>
-  <IsTruncated>false</IsTruncated>
+  <IsTruncated>{{ is_truncated }}</IsTruncated>
   {% for part in parts %}
   <Part>
     <PartNumber>{{ part.name }}</PartNumber>

tests/test_s3/test_s3.py

@@ -2739,6 +2739,141 @@ def test_boto3_multipart_version():
     response["VersionId"].should.should_not.be.none
 
 
+@mock_s3
+def test_boto3_multipart_list_parts_invalid_argument():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    s3.create_bucket(Bucket="mybucket")
+
+    mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key")
+    mpu_id = mpu["UploadId"]
+
+    def get_parts(**kwarg):
+        s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id, **kwarg)
+
+    for value in [-42, 2147483647 + 42]:
+        with pytest.raises(ClientError) as err:
+            get_parts(**{"MaxParts": value})
+        e = err.value.response["Error"]
+        e["Code"].should.equal("InvalidArgument")
+        e["Message"].should.equal(
+            "Argument max-parts must be an integer between 0 and 2147483647"
+        )
+
+        with pytest.raises(ClientError) as err:
+            get_parts(**{"PartNumberMarker": value})
+        e = err.value.response["Error"]
+        e["Code"].should.equal("InvalidArgument")
+        e["Message"].should.equal(
+            "Argument part-number-marker must be an integer between 0 and 2147483647"
+        )
+
+
+@mock_s3
+@reduced_min_part_size
+def test_boto3_multipart_list_parts():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    s3.create_bucket(Bucket="mybucket")
+
+    mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key")
+    mpu_id = mpu["UploadId"]
+
+    parts = []
+    n_parts = 10
+
+    def get_parts_all(i):
+        # Get uploaded parts using default values
+        uploaded_parts = []
+
+        uploaded = s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id,)
+
+        assert uploaded["PartNumberMarker"] == 0
+
+        # Parts content check
+        if i > 0:
+            for part in uploaded["Parts"]:
+                uploaded_parts.append(
+                    {"ETag": part["ETag"], "PartNumber": part["PartNumber"]}
+                )
+            assert uploaded_parts == parts
+
+            next_part_number_marker = uploaded["Parts"][-1]["PartNumber"] + 1
+        else:
+            next_part_number_marker = 0
+
+        assert uploaded["NextPartNumberMarker"] == next_part_number_marker
+
+        assert not uploaded["IsTruncated"]
+
+    def get_parts_by_batch(i):
+        # Get uploaded parts by batch of 2
+        part_number_marker = 0
+        uploaded_parts = []
+
+        while "there are parts":
+            uploaded = s3.list_parts(
+                Bucket="mybucket",
+                Key="the-key",
+                UploadId=mpu_id,
+                PartNumberMarker=part_number_marker,
+                MaxParts=2,
+            )
+
+            assert uploaded["PartNumberMarker"] == part_number_marker
+
+            if i > 0:
+                # We should received maximum 2 parts
+                assert len(uploaded["Parts"]) <= 2
+
+                # Store parts content for the final check
+                for part in uploaded["Parts"]:
+                    uploaded_parts.append(
+                        {"ETag": part["ETag"], "PartNumber": part["PartNumber"]}
+                    )
+
+            # No more parts, get out the loop
+            if not uploaded["IsTruncated"]:
+                break
+
+            # Next parts batch will start with that number
+            part_number_marker = uploaded["NextPartNumberMarker"]
+            assert part_number_marker == i + 1 if len(parts) > i else i
+
+        # Final check: we received all uploaded parts
+        assert uploaded_parts == parts
+
+    # Check ListParts API parameters when no part was uploaded
+    get_parts_all(0)
+    get_parts_by_batch(0)
+
+    for i in range(1, n_parts + 1):
+        part_size = REDUCED_PART_SIZE + i
+        body = b"1" * part_size
+        part = s3.upload_part(
+            Bucket="mybucket",
+            Key="the-key",
+            PartNumber=i,
+            UploadId=mpu_id,
+            Body=body,
+            ContentLength=len(body),
+        )
+        parts.append({"PartNumber": i, "ETag": part["ETag"]})
+
+        # Check ListParts API parameters while there are uploaded parts
+        get_parts_all(i)
+        get_parts_by_batch(i)
+
+    # Check ListParts API parameters when all parts were uploaded
+    get_parts_all(11)
+    get_parts_by_batch(11)
+
+    s3.complete_multipart_upload(
+        Bucket="mybucket",
+        Key="the-key",
+        UploadId=mpu_id,
+        MultipartUpload={"Parts": parts},
+    )
+
+
 @mock_s3
 @reduced_min_part_size
 def test_boto3_multipart_part_size():