From eea67543d154d050ea791f2c0889971eea7260ab Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Thu, 12 Sep 2019 17:54:02 +0800 Subject: [PATCH 01/11] MaxKeys limits the sum of folders and keys --- moto/s3/responses.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ee047a14f..a192cf511 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -458,10 +458,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: result_folders = self._get_results_from_token(result_folders, limit) - if not delimiter: - result_keys, is_truncated, next_continuation_token = self._truncate_result(result_keys, max_keys) - else: - result_folders, is_truncated, next_continuation_token = self._truncate_result(result_folders, max_keys) + tagged_keys = [(key, True) for key in result_keys] + tagged_folders = [(folder, False) for folder in result_folders] + all_keys = tagged_keys + tagged_folders + all_keys.sort() + result_keys, result_folders, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) key_count = len(result_keys) + len(result_folders) @@ -487,16 +488,19 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): continuation_index += 1 return result_keys[continuation_index:] - def _truncate_result(self, result_keys, max_keys): - if len(result_keys) > max_keys: + def _truncate_result(self, all_keys, max_keys): + if len(all_keys) > max_keys: is_truncated = 'true' - result_keys = result_keys[:max_keys] - item = result_keys[-1] + all_keys = all_keys[:max_keys] + item = all_keys[-1][0] next_continuation_token = (item.name if isinstance(item, FakeKey) else item) else: is_truncated = 'false' next_continuation_token = None - return result_keys, is_truncated, next_continuation_token + result_keys, result_folders = [], [] + for (key, is_key) in all_keys: + (result_keys if is_key else result_folders).append(key) + return result_keys, result_folders, is_truncated, next_continuation_token def _bucket_response_put(self, request, body, region_name, bucket_name, querystring): if not request.headers.get('Content-Length'): From d6ef01b9fdfb1521c2adb1cf92ede0933e1648d2 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Thu, 12 Sep 2019 18:40:07 +0800 Subject: [PATCH 02/11] lint --- moto/s3/responses.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index a192cf511..f4640023e 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import re +from itertools import chain + import six from moto.core.utils import str_to_rfc_1123_datetime @@ -458,11 +460,10 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: result_folders = self._get_results_from_token(result_folders, limit) - tagged_keys = [(key, True) for key in result_keys] - tagged_folders = [(folder, False) for folder in result_folders] - all_keys = tagged_keys + tagged_folders - all_keys.sort() - result_keys, result_folders, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) + tagged_keys = ((key, True) for key in result_keys) + tagged_folders = ((folder, False) for folder in result_folders) + sorted_keys = sorted(chain(tagged_keys, tagged_folders)) + result_keys, result_folders, is_truncated, next_continuation_token = self._truncate_result(sorted_keys, max_keys) key_count = len(result_keys) + len(result_folders) @@ -488,17 +489,17 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): continuation_index += 1 return result_keys[continuation_index:] - def _truncate_result(self, all_keys, max_keys): - if len(all_keys) > max_keys: + def _truncate_result(self, sorted_keys, max_keys): + if len(sorted_keys) > max_keys: is_truncated = 'true' - all_keys = all_keys[:max_keys] - item = all_keys[-1][0] + sorted_keys = sorted_keys[:max_keys] + item = sorted_keys[-1][0] next_continuation_token = (item.name if isinstance(item, FakeKey) else item) else: is_truncated = 'false' next_continuation_token = None result_keys, result_folders = [], [] - for (key, is_key) in all_keys: + for (key, is_key) in sorted_keys: (result_keys if is_key else result_folders).append(key) return result_keys, result_folders, is_truncated, next_continuation_token From a36b84b3aa99686a19b19d352983c903039297ba Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 11:35:36 +0800 Subject: [PATCH 03/11] fix MaxKeys in list_objects_v2 --- moto/s3/responses.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f4640023e..e5b5cac0d 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,8 +2,6 @@ from __future__ import unicode_literals import re -from itertools import chain - import six from moto.core.utils import str_to_rfc_1123_datetime @@ -460,10 +458,10 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: result_folders = self._get_results_from_token(result_folders, limit) - tagged_keys = ((key, True) for key in result_keys) - tagged_folders = ((folder, False) for folder in result_folders) - sorted_keys = sorted(chain(tagged_keys, tagged_folders)) - result_keys, result_folders, is_truncated, next_continuation_token = self._truncate_result(sorted_keys, max_keys) + all_keys = [(key, True) for key in result_keys] + [(folder, False) for folder in result_folders] + all_keys.sort(key=lambda tagged_key: tagged_key if isinstance(tagged_key[0], str) else tagged_key[0].name) + truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) + result_keys, result_folders = self._split_truncated_keys(truncated_keys) key_count = len(result_keys) + len(result_folders) @@ -481,6 +479,16 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): start_after=None if continuation_token else start_after ) + def _split_truncated_keys(self, truncated_keys): + result_keys = [] + result_folders = [] + for key in truncated_keys: + if key[1]: + result_keys.append(key[0]) + else: + result_folders.append(key[0]) + return result_keys, result_folders + def _get_results_from_token(self, result_keys, token): continuation_index = 0 for key in result_keys: @@ -489,19 +497,16 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): continuation_index += 1 return result_keys[continuation_index:] - def _truncate_result(self, sorted_keys, max_keys): - if len(sorted_keys) > max_keys: + def _truncate_result(self, result_keys, max_keys): + if len(result_keys) > max_keys: is_truncated = 'true' - sorted_keys = sorted_keys[:max_keys] - item = sorted_keys[-1][0] + result_keys = result_keys[:max_keys] + item = (result_keys[-1][0] if isinstance(result_keys[-1], tuple) else result_keys[-1]) next_continuation_token = (item.name if isinstance(item, FakeKey) else item) else: is_truncated = 'false' next_continuation_token = None - result_keys, result_folders = [], [] - for (key, is_key) in sorted_keys: - (result_keys if is_key else result_folders).append(key) - return result_keys, result_folders, is_truncated, next_continuation_token + return result_keys, is_truncated, next_continuation_token def _bucket_response_put(self, request, body, region_name, bucket_name, querystring): if not request.headers.get('Content-Length'): From 47635dc82e5658e9faf2119ef63cb75d69867000 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 13:33:53 +0800 Subject: [PATCH 04/11] update key of sort --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e5b5cac0d..7005e15df 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -459,7 +459,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): result_folders = self._get_results_from_token(result_folders, limit) all_keys = [(key, True) for key in result_keys] + [(folder, False) for folder in result_folders] - all_keys.sort(key=lambda tagged_key: tagged_key if isinstance(tagged_key[0], str) else tagged_key[0].name) + all_keys.sort(key=lambda tagged_key: tagged_key[0].name if isinstance(tagged_key[0], FakeKey) else tagged_key[0]) truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) result_keys, result_folders = self._split_truncated_keys(truncated_keys) From 59f87e30ba071b383db6bed75995634d9d4bc7ef Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 15:20:24 +0800 Subject: [PATCH 05/11] split truncated keys by type --- moto/s3/responses.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 7005e15df..11c7750a5 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import re +from collections import namedtuple + import six from moto.core.utils import str_to_rfc_1123_datetime @@ -92,6 +94,7 @@ ACTION_MAP = { } +TaggedKey = namedtuple("TaggedKey", ("entity", "is_key")) def parse_key_name(pth): return pth.lstrip("/") @@ -458,8 +461,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: result_folders = self._get_results_from_token(result_folders, limit) - all_keys = [(key, True) for key in result_keys] + [(folder, False) for folder in result_folders] - all_keys.sort(key=lambda tagged_key: tagged_key[0].name if isinstance(tagged_key[0], FakeKey) else tagged_key[0]) + all_keys = result_keys + result_folders + all_keys.sort(key=self._get_key_name) # sort by name, lexicographical order truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) result_keys, result_folders = self._split_truncated_keys(truncated_keys) @@ -479,14 +482,22 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): start_after=None if continuation_token else start_after ) - def _split_truncated_keys(self, truncated_keys): + @staticmethod + def _get_key_name(key): + if isinstance(key, FakeKey): + return key.name + else: + return key + + @staticmethod + def _split_truncated_keys(truncated_keys): result_keys = [] result_folders = [] for key in truncated_keys: - if key[1]: - result_keys.append(key[0]) + if isinstance(key, FakeKey): + result_keys.append(key) else: - result_folders.append(key[0]) + result_folders.append(key) return result_keys, result_folders def _get_results_from_token(self, result_keys, token): @@ -501,7 +512,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): if len(result_keys) > max_keys: is_truncated = 'true' result_keys = result_keys[:max_keys] - item = (result_keys[-1][0] if isinstance(result_keys[-1], tuple) else result_keys[-1]) + item = result_keys[-1] next_continuation_token = (item.name if isinstance(item, FakeKey) else item) else: is_truncated = 'false' From 4946f8b853c7620441be55e86147ccec258dd0d4 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 15:31:57 +0800 Subject: [PATCH 06/11] 'lint' --- moto/s3/responses.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 11c7750a5..582fe8ec7 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,8 +2,6 @@ from __future__ import unicode_literals import re -from collections import namedtuple - import six from moto.core.utils import str_to_rfc_1123_datetime @@ -94,7 +92,6 @@ ACTION_MAP = { } -TaggedKey = namedtuple("TaggedKey", ("entity", "is_key")) def parse_key_name(pth): return pth.lstrip("/") @@ -462,7 +459,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): result_folders = self._get_results_from_token(result_folders, limit) all_keys = result_keys + result_folders - all_keys.sort(key=self._get_key_name) # sort by name, lexicographical order + all_keys.sort(key=self._get_key_name) truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) result_keys, result_folders = self._split_truncated_keys(truncated_keys) From 84715e9a2aadff785f5ba47a931f558ec4182bf7 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 16:46:19 +0800 Subject: [PATCH 07/11] add truncate unite test --- tests/test_s3/test_s3.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 0c0721f01..23e305bcc 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1392,6 +1392,28 @@ def test_boto3_list_objects_v2_fetch_owner(): assert len(owner.keys()) == 2 +@mock_s3 +def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + s3.put_object(Bucket='mybucket', Key="A/a", Body="folder/a") + s3.put_object(Bucket='mybucket', Key="A/aa", Body="folder/aa") + s3.put_object(Bucket='mybucket', Key="A/B/a", Body="nested/folder/a") + s3.put_object(Bucket='mybucket', Key="c", Body="plain c") + + resp = s3.list_objects_v2(Bucket='mybucket', Prefix="A", MaxKeys=2) + + assert "Prefix" in resp + result_keys = [key["Key"] for key in resp["Contents"]] + + # Test truncate combination of keys and folders + assert len(result_keys) == 2 + + # Test lexicographical order + assert "A/B/a" == result_keys[0] + assert "A/a" == result_keys[1] + + @mock_s3 def test_boto3_bucket_create(): s3 = boto3.resource('s3', region_name='us-east-1') From c04c72d435e72d51a78fb4f0a15b9bb39ebca55f Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Mon, 16 Sep 2019 18:09:42 +0800 Subject: [PATCH 08/11] update MaxKeys unite test --- tests/test_s3/test_s3.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 23e305bcc..27005724d 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1401,17 +1401,11 @@ def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): s3.put_object(Bucket='mybucket', Key="A/B/a", Body="nested/folder/a") s3.put_object(Bucket='mybucket', Key="c", Body="plain c") - resp = s3.list_objects_v2(Bucket='mybucket', Prefix="A", MaxKeys=2) - + resp = s3.list_objects_v2(Bucket='mybucket', Prefix="A/", MaxKeys=1, Delimiter="/") assert "Prefix" in resp - result_keys = [key["Key"] for key in resp["Contents"]] - - # Test truncate combination of keys and folders - assert len(result_keys) == 2 - - # Test lexicographical order - assert "A/B/a" == result_keys[0] - assert "A/a" == result_keys[1] + assert "Delimiter" in resp + assert resp["IsTruncated"] is True + assert resp["KeyCount"] != 0 @mock_s3 From 1c36e1e2c5029d16112a121d1dd8bc39cc445fe2 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Tue, 17 Sep 2019 10:42:10 +0800 Subject: [PATCH 09/11] update unit test and fix StartAfter --- moto/s3/responses.py | 13 ++++++------- tests/test_s3/test_s3.py | 22 +++++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 582fe8ec7..4c546c595 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -451,15 +451,14 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): continuation_token = querystring.get('continuation-token', [None])[0] start_after = querystring.get('start-after', [None])[0] + # sort the combination of folders and keys into lexicographical order + all_keys = result_keys + result_folders + all_keys.sort(key=self._get_name) + if continuation_token or start_after: limit = continuation_token or start_after - if not delimiter: - result_keys = self._get_results_from_token(result_keys, limit) - else: - result_folders = self._get_results_from_token(result_folders, limit) + all_keys = self._get_results_from_token(all_keys, limit) - all_keys = result_keys + result_folders - all_keys.sort(key=self._get_key_name) truncated_keys, is_truncated, next_continuation_token = self._truncate_result(all_keys, max_keys) result_keys, result_folders = self._split_truncated_keys(truncated_keys) @@ -480,7 +479,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): ) @staticmethod - def _get_key_name(key): + def _get_name(key): if isinstance(key, FakeKey): return key.name else: diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 27005724d..a8cec737c 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1396,16 +1396,20 @@ def test_boto3_list_objects_v2_fetch_owner(): def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): s3 = boto3.client('s3', region_name='us-east-1') s3.create_bucket(Bucket='mybucket') - s3.put_object(Bucket='mybucket', Key="A/a", Body="folder/a") - s3.put_object(Bucket='mybucket', Key="A/aa", Body="folder/aa") - s3.put_object(Bucket='mybucket', Key="A/B/a", Body="nested/folder/a") - s3.put_object(Bucket='mybucket', Key="c", Body="plain c") + s3.put_object(Bucket='mybucket', Key='1/2', Body='') + s3.put_object(Bucket='mybucket', Key='2', Body='') + s3.put_object(Bucket='mybucket', Key='3/4', Body='') + s3.put_object(Bucket='mybucket', Key='4', Body='') - resp = s3.list_objects_v2(Bucket='mybucket', Prefix="A/", MaxKeys=1, Delimiter="/") - assert "Prefix" in resp - assert "Delimiter" in resp - assert resp["IsTruncated"] is True - assert resp["KeyCount"] != 0 + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Delimiter='/') + assert 'Delimiter' in resp + assert resp['IsTruncated'] is True + assert resp['KeyCount'] == 2 + + last_tail = resp['NextContinuationToken'] + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Delimiter='/', StartAfter=last_tail) + assert resp['KeyCount'] == 2 + assert resp['IsTruncated'] is False @mock_s3 From a466ef2d1ba8796e38b9c1c9f834cfde36afcb14 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Tue, 17 Sep 2019 12:42:33 +0800 Subject: [PATCH 10/11] check key & common prefix in unit test' --- tests/test_s3/test_s3.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index a8cec737c..bbe5e19a3 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1405,11 +1405,19 @@ def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): assert 'Delimiter' in resp assert resp['IsTruncated'] is True assert resp['KeyCount'] == 2 + assert len(resp['Contents']) == 1 + assert resp['Contents'][0]['Key'] == '2' + assert len(resp['CommonPrefixes']) == 1 + assert resp['CommonPrefixes'][0]['Prefix'] == '1/' last_tail = resp['NextContinuationToken'] resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Delimiter='/', StartAfter=last_tail) assert resp['KeyCount'] == 2 assert resp['IsTruncated'] is False + assert len(resp['Contents']) == 1 + assert resp['Contents'][0]['Key'] == '4' + assert len(resp['CommonPrefixes']) == 1 + assert resp['CommonPrefixes'][0]['Prefix'] == '3/' @mock_s3 From d8e69a9a36ecede13120079ba3731ef64b88b2b6 Mon Sep 17 00:00:00 2001 From: Gapex <1377762942@qq.com> Date: Tue, 17 Sep 2019 12:44:48 +0800 Subject: [PATCH 11/11] list with prifix --- tests/test_s3/test_s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index bbe5e19a3..2764ee2c5 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1401,7 +1401,7 @@ def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): s3.put_object(Bucket='mybucket', Key='3/4', Body='') s3.put_object(Bucket='mybucket', Key='4', Body='') - resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Delimiter='/') + resp = s3.list_objects_v2(Bucket='mybucket', Prefix='', MaxKeys=2, Delimiter='/') assert 'Delimiter' in resp assert resp['IsTruncated'] is True assert resp['KeyCount'] == 2 @@ -1411,7 +1411,7 @@ def test_boto3_list_objects_v2_truncate_combined_keys_and_folders(): assert resp['CommonPrefixes'][0]['Prefix'] == '1/' last_tail = resp['NextContinuationToken'] - resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Delimiter='/', StartAfter=last_tail) + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=2, Prefix='', Delimiter='/', StartAfter=last_tail) assert resp['KeyCount'] == 2 assert resp['IsTruncated'] is False assert len(resp['Contents']) == 1