Merge pull request #2417 from Gapex/fix-MaxKeys

MaxKeys limits the sum of folders and keys
This commit is contained in:
Steve Pulec 2019-09-23 21:38:26 -05:00 committed by GitHub
commit 6795a219cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 8 deletions

View File

@ -451,17 +451,16 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
continuation_token = querystring.get('continuation-token', [None])[0]
start_after = querystring.get('start-after', [None])[0]
# sort the combination of folders and keys into lexicographical order
all_keys = result_keys + result_folders
all_keys.sort(key=self._get_name)
if continuation_token or start_after:
limit = continuation_token or start_after
if not delimiter:
result_keys = self._get_results_from_token(result_keys, limit)
else:
result_folders = self._get_results_from_token(result_folders, limit)
all_keys = self._get_results_from_token(all_keys, limit)
if not delimiter:
result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys)
else:
result_folders, is_truncated, next_continuation_token = self._truncate_result(result_folders, max_keys)
truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys)
result_keys, result_folders = self._split_truncated_keys(truncated_keys)
key_count = len(result_keys) + len(result_folders)
@ -479,6 +478,24 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
start_after=None if continuation_token else start_after
)
@staticmethod
def _get_name(key):
if isinstance(key, FakeKey):
return key.name
else:
return key
@staticmethod
def _split_truncated_keys(truncated_keys):
result_keys = []
result_folders = []
for key in truncated_keys:
if isinstance(key, FakeKey):
result_keys.append(key)
else:
result_folders.append(key)
return result_keys, result_folders
def _get_results_from_token(self, result_keys, token):
continuation_index = 0
for key in result_keys:

View File

@ -1392,6 +1392,34 @@ def test_boto3_list_objects_v2_fetch_owner():
assert len(owner.keys()) == 2
@mock_s3
def test_boto3_list_objects_v2_truncate_combined_keys_and_folders():
s3 = boto3.client('s3', region_name='us-east-1')
s3.create_bucket(Bucket='mybucket')
s3.put_object(Bucket='mybucket', Key='1/2', Body='')
s3.put_object(Bucket='mybucket', Key='2', Body='')
s3.put_object(Bucket='mybucket', Key='3/4', Body='')
s3.put_object(Bucket='mybucket', Key='4', Body='')
resp = s3.list_objects_v2(Bucket='mybucket', Prefix='', MaxKeys=2, Delimiter='/')
assert 'Delimiter' in resp
assert resp['IsTruncated'] is True
assert resp['KeyCount'] == 2
assert len(resp['Contents']) == 1
assert resp['Contents'][0]['Key'] == '2'
assert len(resp['CommonPrefixes']) == 1
assert resp['CommonPrefixes'][0]['Prefix'] == '1/'
last_tail = resp['NextContinuationToken']
resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Prefix='', Delimiter='/', StartAfter=last_tail)
assert resp['KeyCount'] == 2
assert resp['IsTruncated'] is False
assert len(resp['Contents']) == 1
assert resp['Contents'][0]['Key'] == '4'
assert len(resp['CommonPrefixes']) == 1
assert resp['CommonPrefixes'][0]['Prefix'] == '3/'
@mock_s3
def test_boto3_bucket_create():
s3 = boto3.resource('s3', region_name='us-east-1')