Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1986 from okomestudio/ts/fix-s3-common-prefix-pagination

Browse Source 
Activate proper pagination for S3 common prefixes
This commit is contained in:
Steve Pulec 2018-12-28 21:22:07 -05:00 committed by GitHub
commit dec665be43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 35 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
moto/s3/responses.py
View File

@ -344,9 +344,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
if continuation_token or start_after: if continuation_token or start_after:
limit = continuation_token or start_after limit = continuation_token or start_after
result_keys = self._get_results_from_token(result_keys, limit) if not delimiter:
result_keys = self._get_results_from_token(result_keys, limit)
else:
result_folders = self._get_results_from_token(result_folders, limit)
result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys) if not delimiter:
result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys)
else:
result_folders, is_truncated, next_continuation_token = self._truncate_result(result_folders, max_keys)
return template.render( return template.render(
bucket=bucket, bucket=bucket,
@ -364,7 +370,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
def _get_results_from_token(self, result_keys, token): def _get_results_from_token(self, result_keys, token):
continuation_index = 0 continuation_index = 0
for key in result_keys: for key in result_keys:
if key.name > token: if (key.name if isinstance(key, FakeKey) else key) > token:
break break
continuation_index += 1 continuation_index += 1
return result_keys[continuation_index:] return result_keys[continuation_index:]
@ -373,7 +379,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
if len(result_keys) > max_keys: if len(result_keys) > max_keys:
is_truncated = 'true' is_truncated = 'true'
result_keys = result_keys[:max_keys] result_keys = result_keys[:max_keys]
next_continuation_token = result_keys[-1].name item = result_keys[-1]
next_continuation_token = (item.name if isinstance(item, FakeKey) else item)
else: else:
is_truncated = 'false' is_truncated = 'false'
next_continuation_token = None next_continuation_token = None

24
tests/test_s3/test_s3.py
View File

@ -1172,6 +1172,30 @@ def test_boto3_list_keys_xml_escaped():
assert 'Owner' not in resp['Contents'][0] assert 'Owner' not in resp['Contents'][0]
@mock_s3
def test_boto3_list_objects_v2_common_prefix_pagination():
s3 = boto3.client('s3', region_name='us-east-1')
s3.create_bucket(Bucket='mybucket')
max_keys = 1
keys = ['test/{i}/{i}'.format(i=i) for i in range(3)]
for key in keys:
s3.put_object(Bucket='mybucket', Key=key, Body=b'v')
prefixes = []
args = {"Bucket": 'mybucket', "Delimiter": "/", "Prefix": "test/", "MaxKeys": max_keys}
resp = {"IsTruncated": True}
while resp.get("IsTruncated", False):
if "NextContinuationToken" in resp:
args["ContinuationToken"] = resp["NextContinuationToken"]
resp = s3.list_objects_v2(**args)
if "CommonPrefixes" in resp:
assert len(resp["CommonPrefixes"]) == max_keys
prefixes.extend(i["Prefix"] for i in resp["CommonPrefixes"])
assert prefixes == [k[:k.rindex('/') + 1] for k in keys]
@mock_s3 @mock_s3
def test_boto3_list_objects_v2_truncated_response(): def test_boto3_list_objects_v2_truncated_response():
s3 = boto3.client('s3', region_name='us-east-1') s3 = boto3.client('s3', region_name='us-east-1')