diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ac1533eb0..d6855265e 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -215,6 +215,8 @@ class ResponseObject(_TemplateEnvironmentMixin): delimiter='', is_truncated='false', ) + elif querystring.get('list-type', [None])[0] == '2': + return 200, headers, self._handle_list_objects_v2(bucket_name, querystring) bucket = self.backend.get_bucket(bucket_name) prefix = querystring.get('prefix', [None])[0] @@ -229,6 +231,49 @@ class ResponseObject(_TemplateEnvironmentMixin): result_folders=result_folders ) + def _handle_list_objects_v2(self, bucket_name, querystring): + template = self.response_template(S3_BUCKET_GET_RESPONSE_V2) + bucket = self.backend.get_bucket(bucket_name) + + prefix = querystring.get('prefix', [None])[0] + delimiter = querystring.get('delimiter', [None])[0] + result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter) + + fetch_owner = querystring.get('fetch-owner', [False])[0] + max_keys = int(querystring.get('max-keys', [1000])[0]) + continuation_token = querystring.get('continuation-token', [None])[0] + start_after = querystring.get('start-after', [None])[0] + + if continuation_token or start_after: + limit = continuation_token or start_after + continuation_index = 0 + for key in result_keys: + if key.name > limit: + break + continuation_index += 1 + result_keys = result_keys[continuation_index:] + + if len(result_keys) > max_keys: + is_truncated = 'true' + result_keys = result_keys[:max_keys] + next_continuation_token = result_keys[-1].name + else: + is_truncated = 'false' + next_continuation_token = None + + return template.render( + bucket=bucket, + prefix=prefix or '', + delimiter=delimiter, + result_keys=result_keys, + result_folders=result_folders, + fetch_owner=fetch_owner, + max_keys=max_keys, + is_truncated=is_truncated, + next_continuation_token=next_continuation_token, + start_after=None if continuation_token else start_after + ) + def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers): if 'versioning' in querystring: ver = re.search('([A-Za-z]+)', body) @@ -636,6 +681,46 @@ S3_BUCKET_GET_RESPONSE = """ {% endif %} """ +S3_BUCKET_GET_RESPONSE_V2 = """ + + {{ bucket.name }} + {{ prefix }} + {{ max_keys }} + {{ result_keys | length }} +{% if delimiter %} + {{ delimiter }} +{% endif %} + {{ is_truncated }} +{% if next_continuation_token %} + {{ next_continuation_token }} +{% endif %} +{% if start_after %} + {{ start_after }} +{% endif %} + {% for key in result_keys %} + + {{ key.name }} + {{ key.last_modified_ISO8601 }} + {{ key.etag }} + {{ key.size }} + {{ key.storage_class }} + {% if fetch_owner %} + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + {% endif %} + + {% endfor %} + {% if delimiter %} + {% for folder in result_folders %} + + {{ folder }} + + {% endfor %} + {% endif %} + """ + S3_BUCKET_CREATE_RESPONSE = """ {{ bucket.name }} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 0d8f7cb49..4990d7324 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -995,8 +995,129 @@ def test_boto3_list_keys_xml_escaped(): s3.create_bucket(Bucket='mybucket') key_name = 'Q&A.txt' s3.put_object(Bucket='mybucket', Key=key_name, Body=b'is awesome') + resp = s3.list_objects_v2(Bucket='mybucket', Prefix=key_name) + assert resp['Contents'][0]['Key'] == key_name + assert resp['KeyCount'] == 1 + assert resp['MaxKeys'] == 1000 + assert resp['Prefix'] == key_name + assert resp['IsTruncated'] == False + assert 'Delimiter' not in resp + assert 'StartAfter' not in resp + assert 'NextContinuationToken' not in resp + assert 'Owner' not in resp['Contents'][0] + + +@mock_s3 +def test_boto3_list_objects_v2_truncated_response(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + s3.put_object(Bucket='mybucket', Key='one', Body=b'1') + s3.put_object(Bucket='mybucket', Key='two', Body=b'22') + s3.put_object(Bucket='mybucket', Key='three', Body=b'333') + + # First list + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1) + listed_object = resp['Contents'][0] + + assert listed_object['Key'] == 'one' + assert resp['MaxKeys'] == 1 + assert resp['Prefix'] == '' + assert resp['KeyCount'] == 1 + assert resp['IsTruncated'] == True + assert 'Delimiter' not in resp + assert 'StartAfter' not in resp + assert 'Owner' not in listed_object # owner info was not requested + + next_token = resp['NextContinuationToken'] + + + # Second list + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, ContinuationToken=next_token) + listed_object = resp['Contents'][0] + + assert listed_object['Key'] == 'three' + assert resp['MaxKeys'] == 1 + assert resp['Prefix'] == '' + assert resp['KeyCount'] == 1 + assert resp['IsTruncated'] == True + assert 'Delimiter' not in resp + assert 'StartAfter' not in resp + assert 'Owner' not in listed_object + + next_token = resp['NextContinuationToken'] + + + # Third list + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, ContinuationToken=next_token) + listed_object = resp['Contents'][0] + + assert listed_object['Key'] == 'two' + assert resp['MaxKeys'] == 1 + assert resp['Prefix'] == '' + assert resp['KeyCount'] == 1 + assert resp['IsTruncated'] == False + assert 'Delimiter' not in resp + assert 'Owner' not in listed_object + assert 'StartAfter' not in resp + assert 'NextContinuationToken' not in resp + + +@mock_s3 +def test_boto3_list_objects_v2_truncated_response_start_after(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + s3.put_object(Bucket='mybucket', Key='one', Body=b'1') + s3.put_object(Bucket='mybucket', Key='two', Body=b'22') + s3.put_object(Bucket='mybucket', Key='three', Body=b'333') + + # First list + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, StartAfter='one') + listed_object = resp['Contents'][0] + + assert listed_object['Key'] == 'three' + assert resp['MaxKeys'] == 1 + assert resp['Prefix'] == '' + assert resp['KeyCount'] == 1 + assert resp['IsTruncated'] == True + assert resp['StartAfter'] == 'one' + assert 'Delimiter' not in resp + assert 'Owner' not in listed_object + + next_token = resp['NextContinuationToken'] + + # Second list + # The ContinuationToken must take precedence over StartAfter. + resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, StartAfter='one', + ContinuationToken=next_token) + listed_object = resp['Contents'][0] + + assert listed_object['Key'] == 'two' + assert resp['MaxKeys'] == 1 + assert resp['Prefix'] == '' + assert resp['KeyCount'] == 1 + assert resp['IsTruncated'] == False + # When ContinuationToken is given, StartAfter is ignored. This also means + # AWS does not return it in the response. + assert 'StartAfter' not in resp + assert 'Delimiter' not in resp + assert 'Owner' not in listed_object + + +@mock_s3 +def test_boto3_list_objects_v2_fetch_owner(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + s3.put_object(Bucket='mybucket', Key='one', Body=b'11') + + resp = s3.list_objects_v2(Bucket='mybucket', FetchOwner=True) + owner = resp['Contents'][0]['Owner'] + + assert 'ID' in owner + assert 'DisplayName' in owner + assert len(owner.keys()) == 2 + @mock_s3 def test_boto3_bucket_create():