Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

S3 ListParts API: use MaxParts parameter (#3658)

Browse Source
This commit is contained in:
Bert Blommers 2021-08-28 07:38:16 +01:00 committed by GitHub
parent 6b960e0d4f
commit 0317c502f0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 199 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
AUTHORS.md
View File

@ -58,4 +58,5 @@ Moto is written by Steve Pulec with contributions from:
* [Craig Anderson](https://github.com/craiga)
* [Robert Lewis](https://github.com/ralewis85)
* [Kyle Jones](https://github.com/Kerl1310)
* [Ariel Beck](https://github.com/arielb135)
* [Mickaël Schoentgen](https://github.com/BoboTiG)
* [Ariel Beck](https://github.com/arielb135)

3
IMPLEMENTATION_COVERAGE.md
View File

@ -9485,6 +9485,9 @@
- [ ] list_objects
- [ ] list_objects_v2
- [ ] list_parts
- [X] list_objects
- [X] list_objects_v2
- [X] list_parts
- [X] put_bucket_accelerate_configuration
- [X] put_bucket_acl
- [ ] put_bucket_analytics_configuration

10
moto/s3/exceptions.py
View File

@ -237,6 +237,16 @@ class CrossLocationLoggingProhibitted(S3ClientError):
)
class InvalidMaxPartArgument(S3ClientError):
code = 400
def __init__(self, arg, min_val, max_val):
error = "Argument {} must be an integer between {} and {}".format(
arg, min_val, max_val
)
super(InvalidMaxPartArgument, self).__init__("InvalidArgument", error)
class InvalidNotificationARN(S3ClientError):
code = 400

26
moto/s3/models.py
View File

@ -369,9 +369,11 @@ class FakeMultipart(BaseModel):
insort(self.partlist, part_id)
return key
def list_parts(self):
def list_parts(self, part_number_marker, max_parts):
for part_id in self.partlist:
yield self.parts[part_id]
part = self.parts[part_id]
if part_number_marker <= part.name < part_number_marker + max_parts:
yield part
class FakeGrantee(BaseModel):
@ -1712,6 +1714,20 @@ class S3Backend(BaseBackend):
raise NoSuchUpload(upload_id=multipart_id)
del bucket.multiparts[multipart_id]
def list_multipart(
self, bucket_name, multipart_id, part_number_marker=0, max_parts=1000
):
bucket = self.get_bucket(bucket_name)
if multipart_id not in bucket.multiparts:
raise NoSuchUpload(upload_id=multipart_id)
return list(
bucket.multiparts[multipart_id].list_parts(part_number_marker, max_parts)
)
def is_truncated(self, bucket_name, multipart_id, next_part_number_marker):
bucket = self.get_bucket(bucket_name)
return len(bucket.multiparts[multipart_id].parts) >= next_part_number_marker
def create_multipart_upload(self, bucket_name, key_name, metadata, storage_type):
multipart = FakeMultipart(key_name, metadata)
multipart.storage = storage_type
@ -1728,12 +1744,6 @@ class S3Backend(BaseBackend):
del bucket.multiparts[multipart_id]
return multipart, value, etag
def list_multipart(self, bucket_name, multipart_id):
bucket = self.get_bucket(bucket_name)
if multipart_id not in bucket.multiparts:
raise NoSuchUpload(upload_id=multipart_id)
return list(bucket.multiparts[multipart_id].list_parts())
def get_all_multiparts(self, bucket_name):
bucket = self.get_bucket(bucket_name)
return bucket.multiparts

37
moto/s3/responses.py
View File

@ -45,6 +45,7 @@ from .exceptions import (
MissingBucket,
MissingKey,
MissingVersion,
InvalidMaxPartArgument,
InvalidPartOrder,
MalformedXML,
MalformedACLError,
@ -1240,7 +1241,28 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
response_headers = {}
if query.get("uploadId"):
upload_id = query["uploadId"][0]
parts = self.backend.list_multipart(bucket_name, upload_id)
# 0 <= PartNumberMarker <= 2,147,483,647
part_number_marker = int(query.get("part-number-marker", [0])[0])
if not (0 <= part_number_marker <= 2147483647):
raise InvalidMaxPartArgument("part-number-marker", 0, 2147483647)
# 0 <= MaxParts <= 2,147,483,647 (default is 1,000)
max_parts = int(query.get("max-parts", [1000])[0])
if not (0 <= max_parts <= 2147483647):
raise InvalidMaxPartArgument("max-parts", 0, 2147483647)
parts = self.backend.list_multipart(
bucket_name,
upload_id,
part_number_marker=part_number_marker,
max_parts=max_parts,
)
next_part_number_marker = parts[-1].name + 1 if parts else 0
is_truncated = parts and self.backend.is_truncated(
bucket_name, upload_id, next_part_number_marker
)
template = self.response_template(S3_MULTIPART_LIST_RESPONSE)
return (
200,
@ -1249,8 +1271,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
bucket_name=bucket_name,
key_name=key_name,
upload_id=upload_id,
count=len(parts),
is_truncated=str(is_truncated).lower(),
max_parts=max_parts,
next_part_number_marker=next_part_number_marker,
parts=parts,
part_number_marker=part_number_marker,
),
)
version_id = query.get("versionId", [None])[0]
@ -2358,10 +2383,10 @@ S3_MULTIPART_LIST_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
<ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>
<DisplayName>webfile</DisplayName>
</Owner>
<PartNumberMarker>1</PartNumberMarker>
<NextPartNumberMarker>{{ count }}</NextPartNumberMarker>
<MaxParts>{{ count }}</MaxParts>
<IsTruncated>false</IsTruncated>
<PartNumberMarker>{{ part_number_marker }}</PartNumberMarker>
<NextPartNumberMarker>{{ next_part_number_marker }}</NextPartNumberMarker>
<MaxParts>{{ max_parts }}</MaxParts>
<IsTruncated>{{ is_truncated }}</IsTruncated>
{% for part in parts %}
<Part>
<PartNumber>{{ part.name }}</PartNumber>

135
tests/test_s3/test_s3.py
View File

@ -2739,6 +2739,141 @@ def test_boto3_multipart_version():
response["VersionId"].should.should_not.be.none
@mock_s3
def test_boto3_multipart_list_parts_invalid_argument():
s3 = boto3.client("s3", region_name="us-east-1")
s3.create_bucket(Bucket="mybucket")
mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key")
mpu_id = mpu["UploadId"]
def get_parts(**kwarg):
s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id, **kwarg)
for value in [-42, 2147483647 + 42]:
with pytest.raises(ClientError) as err:
get_parts(**{"MaxParts": value})
e = err.value.response["Error"]
e["Code"].should.equal("InvalidArgument")
e["Message"].should.equal(
"Argument max-parts must be an integer between 0 and 2147483647"
)
with pytest.raises(ClientError) as err:
get_parts(**{"PartNumberMarker": value})
e = err.value.response["Error"]
e["Code"].should.equal("InvalidArgument")
e["Message"].should.equal(
"Argument part-number-marker must be an integer between 0 and 2147483647"
)
@mock_s3
@reduced_min_part_size
def test_boto3_multipart_list_parts():
s3 = boto3.client("s3", region_name="us-east-1")
s3.create_bucket(Bucket="mybucket")
mpu = s3.create_multipart_upload(Bucket="mybucket", Key="the-key")
mpu_id = mpu["UploadId"]
parts = []
n_parts = 10
def get_parts_all(i):
# Get uploaded parts using default values
uploaded_parts = []
uploaded = s3.list_parts(Bucket="mybucket", Key="the-key", UploadId=mpu_id,)
assert uploaded["PartNumberMarker"] == 0
# Parts content check
if i > 0:
for part in uploaded["Parts"]:
uploaded_parts.append(
{"ETag": part["ETag"], "PartNumber": part["PartNumber"]}
)
assert uploaded_parts == parts
next_part_number_marker = uploaded["Parts"][-1]["PartNumber"] + 1
else:
next_part_number_marker = 0
assert uploaded["NextPartNumberMarker"] == next_part_number_marker
assert not uploaded["IsTruncated"]
def get_parts_by_batch(i):
# Get uploaded parts by batch of 2
part_number_marker = 0
uploaded_parts = []
while "there are parts":
uploaded = s3.list_parts(
Bucket="mybucket",
Key="the-key",
UploadId=mpu_id,
PartNumberMarker=part_number_marker,
MaxParts=2,
)
assert uploaded["PartNumberMarker"] == part_number_marker
if i > 0:
# We should received maximum 2 parts
assert len(uploaded["Parts"]) <= 2
# Store parts content for the final check
for part in uploaded["Parts"]:
uploaded_parts.append(
{"ETag": part["ETag"], "PartNumber": part["PartNumber"]}
)
# No more parts, get out the loop
if not uploaded["IsTruncated"]:
break
# Next parts batch will start with that number
part_number_marker = uploaded["NextPartNumberMarker"]
assert part_number_marker == i + 1 if len(parts) > i else i
# Final check: we received all uploaded parts
assert uploaded_parts == parts
# Check ListParts API parameters when no part was uploaded
get_parts_all(0)
get_parts_by_batch(0)
for i in range(1, n_parts + 1):
part_size = REDUCED_PART_SIZE + i
body = b"1" * part_size
part = s3.upload_part(
Bucket="mybucket",
Key="the-key",
PartNumber=i,
UploadId=mpu_id,
Body=body,
ContentLength=len(body),
)
parts.append({"PartNumber": i, "ETag": part["ETag"]})
# Check ListParts API parameters while there are uploaded parts
get_parts_all(i)
get_parts_by_batch(i)
# Check ListParts API parameters when all parts were uploaded
get_parts_all(11)
get_parts_by_batch(11)
s3.complete_multipart_upload(
Bucket="mybucket",
Key="the-key",
UploadId=mpu_id,
MultipartUpload={"Parts": parts},
)
@mock_s3
@reduced_min_part_size
def test_boto3_multipart_part_size():