diff --git a/AUTHORS.md b/AUTHORS.md index 31a348ce4..5f7d047ce 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -58,4 +58,5 @@ Moto is written by Steve Pulec with contributions from: * [Craig Anderson](https://github.com/craiga) * [Robert Lewis](https://github.com/ralewis85) * [Kyle Jones](https://github.com/Kerl1310) -* [Ariel Beck](https://github.com/arielb135) \ No newline at end of file +* [Mickaƫl Schoentgen](https://github.com/BoboTiG) +* [Ariel Beck](https://github.com/arielb135) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 67c0e997a..e453af375 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -9485,6 +9485,9 @@ - [ ] list_objects - [ ] list_objects_v2 - [ ] list_parts +- [X] list_objects +- [X] list_objects_v2 +- [X] list_parts - [X] put_bucket_accelerate_configuration - [X] put_bucket_acl - [ ] put_bucket_analytics_configuration diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 5df286b3c..3abda1f78 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -237,6 +237,16 @@ class CrossLocationLoggingProhibitted(S3ClientError): ) +class InvalidMaxPartArgument(S3ClientError): + code = 400 + + def __init__(self, arg, min_val, max_val): + error = "Argument {} must be an integer between {} and {}".format( + arg, min_val, max_val + ) + super(InvalidMaxPartArgument, self).__init__("InvalidArgument", error) + + class InvalidNotificationARN(S3ClientError): code = 400 diff --git a/moto/s3/models.py b/moto/s3/models.py index e606198b8..8e7bb9d3f 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -369,9 +369,11 @@ class FakeMultipart(BaseModel): insort(self.partlist, part_id) return key - def list_parts(self): + def list_parts(self, part_number_marker, max_parts): for part_id in self.partlist: - yield self.parts[part_id] + part = self.parts[part_id] + if part_number_marker <= part.name < part_number_marker + max_parts: + yield part class FakeGrantee(BaseModel): @@ -1712,6 +1714,20 @@ class S3Backend(BaseBackend): raise NoSuchUpload(upload_id=multipart_id) del bucket.multiparts[multipart_id] + def list_multipart( + self, bucket_name, multipart_id, part_number_marker=0, max_parts=1000 + ): + bucket = self.get_bucket(bucket_name) + if multipart_id not in bucket.multiparts: + raise NoSuchUpload(upload_id=multipart_id) + return list( + bucket.multiparts[multipart_id].list_parts(part_number_marker, max_parts) + ) + + def is_truncated(self, bucket_name, multipart_id, next_part_number_marker): + bucket = self.get_bucket(bucket_name) + return len(bucket.multiparts[multipart_id].parts) >= next_part_number_marker + def create_multipart_upload(self, bucket_name, key_name, metadata, storage_type): multipart = FakeMultipart(key_name, metadata) multipart.storage = storage_type @@ -1728,12 +1744,6 @@ class S3Backend(BaseBackend): del bucket.multiparts[multipart_id] return multipart, value, etag - def list_multipart(self, bucket_name, multipart_id): - bucket = self.get_bucket(bucket_name) - if multipart_id not in bucket.multiparts: - raise NoSuchUpload(upload_id=multipart_id) - return list(bucket.multiparts[multipart_id].list_parts()) - def get_all_multiparts(self, bucket_name): bucket = self.get_bucket(bucket_name) return bucket.multiparts diff --git a/moto/s3/responses.py b/moto/s3/responses.py index db620b0a6..895c01259 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -45,6 +45,7 @@ from .exceptions import ( MissingBucket, MissingKey, MissingVersion, + InvalidMaxPartArgument, InvalidPartOrder, MalformedXML, MalformedACLError, @@ -1240,7 +1241,28 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_headers = {} if query.get("uploadId"): upload_id = query["uploadId"][0] - parts = self.backend.list_multipart(bucket_name, upload_id) + + # 0 <= PartNumberMarker <= 2,147,483,647 + part_number_marker = int(query.get("part-number-marker", [0])[0]) + if not (0 <= part_number_marker <= 2147483647): + raise InvalidMaxPartArgument("part-number-marker", 0, 2147483647) + + # 0 <= MaxParts <= 2,147,483,647 (default is 1,000) + max_parts = int(query.get("max-parts", [1000])[0]) + if not (0 <= max_parts <= 2147483647): + raise InvalidMaxPartArgument("max-parts", 0, 2147483647) + + parts = self.backend.list_multipart( + bucket_name, + upload_id, + part_number_marker=part_number_marker, + max_parts=max_parts, + ) + next_part_number_marker = parts[-1].name + 1 if parts else 0 + is_truncated = parts and self.backend.is_truncated( + bucket_name, upload_id, next_part_number_marker + ) + template = self.response_template(S3_MULTIPART_LIST_RESPONSE) return ( 200, @@ -1249,8 +1271,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): bucket_name=bucket_name, key_name=key_name, upload_id=upload_id, - count=len(parts), + is_truncated=str(is_truncated).lower(), + max_parts=max_parts, + next_part_number_marker=next_part_number_marker, parts=parts, + part_number_marker=part_number_marker, ), ) version_id = query.get("versionId", [None])[0] @@ -2358,10 +2383,10 @@ S3_MULTIPART_LIST_RESPONSE = """ 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a webfile - 1 - {{ count }} - {{ count }} - false + {{ part_number_marker }} + {{ next_part_number_marker }} + {{ max_parts }} + {{ is_truncated }} {% for part in parts %} {{ part.name }} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index e6721689c..540be8239 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2739,6 +2739,141 @@ def test_boto3_multipart_version(): response["VersionId"].should.should_not.be.none +@mock_s3 +def test_boto3_multipart_list_parts_invalid_argument(): + s3 = boto3.client("s3", region_name="us-east-1") + s3.create_bucket(Bucket="mybucket") + + mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key") + mpu_id = mpu["UploadId"] + + def get_parts(**kwarg): + s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id, **kwarg) + + for value in [-42, 2147483647 + 42]: + with pytest.raises(ClientError) as err: + get_parts(**{"MaxParts": value}) + e = err.value.response["Error"] + e["Code"].should.equal("InvalidArgument") + e["Message"].should.equal( + "Argument max-parts must be an integer between 0 and 2147483647" + ) + + with pytest.raises(ClientError) as err: + get_parts(**{"PartNumberMarker": value}) + e = err.value.response["Error"] + e["Code"].should.equal("InvalidArgument") + e["Message"].should.equal( + "Argument part-number-marker must be an integer between 0 and 2147483647" + ) + + +@mock_s3 +@reduced_min_part_size +def test_boto3_multipart_list_parts(): + s3 = boto3.client("s3", region_name="us-east-1") + s3.create_bucket(Bucket="mybucket") + + mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key") + mpu_id = mpu["UploadId"] + + parts = [] + n_parts = 10 + + def get_parts_all(i): + # Get uploaded parts using default values + uploaded_parts = [] + + uploaded = s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id,) + + assert uploaded["PartNumberMarker"] == 0 + + # Parts content check + if i > 0: + for part in uploaded["Parts"]: + uploaded_parts.append( + {"ETag": part["ETag"], "PartNumber": part["PartNumber"]} + ) + assert uploaded_parts == parts + + next_part_number_marker = uploaded["Parts"][-1]["PartNumber"] + 1 + else: + next_part_number_marker = 0 + + assert uploaded["NextPartNumberMarker"] == next_part_number_marker + + assert not uploaded["IsTruncated"] + + def get_parts_by_batch(i): + # Get uploaded parts by batch of 2 + part_number_marker = 0 + uploaded_parts = [] + + while "there are parts": + uploaded = s3.list_parts( + Bucket="mybucket", + Key="the-key", + UploadId=mpu_id, + PartNumberMarker=part_number_marker, + MaxParts=2, + ) + + assert uploaded["PartNumberMarker"] == part_number_marker + + if i > 0: + # We should received maximum 2 parts + assert len(uploaded["Parts"]) <= 2 + + # Store parts content for the final check + for part in uploaded["Parts"]: + uploaded_parts.append( + {"ETag": part["ETag"], "PartNumber": part["PartNumber"]} + ) + + # No more parts, get out the loop + if not uploaded["IsTruncated"]: + break + + # Next parts batch will start with that number + part_number_marker = uploaded["NextPartNumberMarker"] + assert part_number_marker == i + 1 if len(parts) > i else i + + # Final check: we received all uploaded parts + assert uploaded_parts == parts + + # Check ListParts API parameters when no part was uploaded + get_parts_all(0) + get_parts_by_batch(0) + + for i in range(1, n_parts + 1): + part_size = REDUCED_PART_SIZE + i + body = b"1" * part_size + part = s3.upload_part( + Bucket="mybucket", + Key="the-key", + PartNumber=i, + UploadId=mpu_id, + Body=body, + ContentLength=len(body), + ) + parts.append({"PartNumber": i, "ETag": part["ETag"]}) + + # Check ListParts API parameters while there are uploaded parts + get_parts_all(i) + get_parts_by_batch(i) + + # Check ListParts API parameters when all parts were uploaded + get_parts_all(11) + get_parts_by_batch(11) + + s3.complete_multipart_upload( + Bucket="mybucket", + Key="the-key", + UploadId=mpu_id, + MultipartUpload={"Parts": parts}, + ) + + @mock_s3 @reduced_min_part_size def test_boto3_multipart_part_size():