Implement list_objects_v2() for S3 buckets (#814)

This adds/fixes the following things:
- Add missing KeyCount in result (fixes #734).
- Do not hard code MaxKeys to 1000.
- Truncate result if it has more than MaxKeys items. Set IsTruncated
    and NextContinuationToken accordingly.
- Support the StartAfter parameter.
- Return Owner information only when FetchOwner=True is given.
- "Prefix" in response is now "" instead of None when omitted in request.
- "Delimiter" is now omitted from response when not given in request.

moto/s3/responses.py

@@ -215,6 +215,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
                 delimiter='',
                 is_truncated='false',
             )
+        elif querystring.get('list-type', [None])[0] == '2':
+            return 200, headers, self._handle_list_objects_v2(bucket_name, querystring)
 
         bucket = self.backend.get_bucket(bucket_name)
         prefix = querystring.get('prefix', [None])[0]
@@ -229,6 +231,49 @@ class ResponseObject(_TemplateEnvironmentMixin):
             result_folders=result_folders
         )
 
+    def _handle_list_objects_v2(self, bucket_name, querystring):
+        template = self.response_template(S3_BUCKET_GET_RESPONSE_V2)
+        bucket = self.backend.get_bucket(bucket_name)
+
+        prefix = querystring.get('prefix', [None])[0]
+        delimiter = querystring.get('delimiter', [None])[0]
+        result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
+
+        fetch_owner = querystring.get('fetch-owner', [False])[0]
+        max_keys = int(querystring.get('max-keys', [1000])[0])
+        continuation_token = querystring.get('continuation-token', [None])[0]
+        start_after = querystring.get('start-after', [None])[0]
+
+        if continuation_token or start_after:
+            limit = continuation_token or start_after
+            continuation_index = 0
+            for key in result_keys:
+                if key.name > limit:
+                    break
+                continuation_index += 1
+            result_keys = result_keys[continuation_index:]
+
+        if len(result_keys) > max_keys:
+            is_truncated = 'true'
+            result_keys = result_keys[:max_keys]
+            next_continuation_token = result_keys[-1].name
+        else:
+            is_truncated = 'false'
+            next_continuation_token = None
+
+        return template.render(
+            bucket=bucket,
+            prefix=prefix or '',
+            delimiter=delimiter,
+            result_keys=result_keys,
+            result_folders=result_folders,
+            fetch_owner=fetch_owner,
+            max_keys=max_keys,
+            is_truncated=is_truncated,
+            next_continuation_token=next_continuation_token,
+            start_after=None if continuation_token else start_after
+        )
+
     def _bucket_response_put(self, request, body, region_name, bucket_name, querystring, headers):
         if 'versioning' in querystring:
             ver = re.search('<Status>([A-Za-z]+)</Status>', body)
@@ -636,6 +681,46 @@ S3_BUCKET_GET_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
   {% endif %}
   </ListBucketResult>"""
 
+S3_BUCKET_GET_RESPONSE_V2 = """<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>{{ bucket.name }}</Name>
+  <Prefix>{{ prefix }}</Prefix>
+  <MaxKeys>{{ max_keys }}</MaxKeys>
+  <KeyCount>{{ result_keys | length }}</KeyCount>
+{% if delimiter %}
+  <Delimiter>{{ delimiter }}</Delimiter>
+{% endif %}
+  <IsTruncated>{{ is_truncated }}</IsTruncated>
+{% if next_continuation_token %}
+  <NextContinuationToken>{{ next_continuation_token }}</NextContinuationToken>
+{% endif %}
+{% if start_after %}
+  <StartAfter>{{ start_after }}</StartAfter>
+{% endif %}
+  {% for key in result_keys %}
+    <Contents>
+      <Key>{{ key.name }}</Key>
+      <LastModified>{{ key.last_modified_ISO8601 }}</LastModified>
+      <ETag>{{ key.etag }}</ETag>
+      <Size>{{ key.size }}</Size>
+      <StorageClass>{{ key.storage_class }}</StorageClass>
+      {% if fetch_owner %}
+      <Owner>
+        <ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>
+        <DisplayName>webfile</DisplayName>
+      </Owner>
+      {% endif %}
+    </Contents>
+  {% endfor %}
+  {% if delimiter %}
+    {% for folder in result_folders %}
+      <CommonPrefixes>
+        <Prefix>{{ folder }}</Prefix>
+      </CommonPrefixes>
+    {% endfor %}
+  {% endif %}
+  </ListBucketResult>"""
+
 S3_BUCKET_CREATE_RESPONSE = """<CreateBucketResponse xmlns="http://s3.amazonaws.com/doc/2006-03-01">
   <CreateBucketResponse>
     <Bucket>{{ bucket.name }}</Bucket>

tests/test_s3/test_s3.py

@@ -995,8 +995,129 @@ def test_boto3_list_keys_xml_escaped():
     s3.create_bucket(Bucket='mybucket')
     key_name = 'Q&A.txt'
     s3.put_object(Bucket='mybucket', Key=key_name, Body=b'is awesome')
+
     resp = s3.list_objects_v2(Bucket='mybucket', Prefix=key_name)
+
     assert resp['Contents'][0]['Key'] == key_name
+    assert resp['KeyCount'] == 1
+    assert resp['MaxKeys'] == 1000
+    assert resp['Prefix'] == key_name
+    assert resp['IsTruncated'] == False
+    assert 'Delimiter' not in resp
+    assert 'StartAfter' not in resp
+    assert 'NextContinuationToken' not in resp
+    assert 'Owner' not in resp['Contents'][0]
+
+
+@mock_s3
+def test_boto3_list_objects_v2_truncated_response():
+    s3 = boto3.client('s3', region_name='us-east-1')
+    s3.create_bucket(Bucket='mybucket')
+    s3.put_object(Bucket='mybucket', Key='one', Body=b'1')
+    s3.put_object(Bucket='mybucket', Key='two', Body=b'22')
+    s3.put_object(Bucket='mybucket', Key='three', Body=b'333')
+
+    # First list
+    resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1)
+    listed_object = resp['Contents'][0]
+
+    assert listed_object['Key'] == 'one'
+    assert resp['MaxKeys'] == 1
+    assert resp['Prefix'] == ''
+    assert resp['KeyCount'] == 1
+    assert resp['IsTruncated'] == True
+    assert 'Delimiter' not in resp
+    assert 'StartAfter' not in resp
+    assert 'Owner' not in listed_object # owner info was not requested
+
+    next_token = resp['NextContinuationToken']
+
+
+    # Second list
+    resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, ContinuationToken=next_token)
+    listed_object = resp['Contents'][0]
+
+    assert listed_object['Key'] == 'three'
+    assert resp['MaxKeys'] == 1
+    assert resp['Prefix'] == ''
+    assert resp['KeyCount'] == 1
+    assert resp['IsTruncated'] == True
+    assert 'Delimiter' not in resp
+    assert 'StartAfter' not in resp
+    assert 'Owner' not in listed_object
+
+    next_token = resp['NextContinuationToken']
+
+
+    # Third list
+    resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, ContinuationToken=next_token)
+    listed_object = resp['Contents'][0]
+
+    assert listed_object['Key'] == 'two'
+    assert resp['MaxKeys'] == 1
+    assert resp['Prefix'] == ''
+    assert resp['KeyCount'] == 1
+    assert resp['IsTruncated'] == False
+    assert 'Delimiter' not in resp
+    assert 'Owner' not in listed_object
+    assert 'StartAfter' not in resp
+    assert 'NextContinuationToken' not in resp
+
+
+@mock_s3
+def test_boto3_list_objects_v2_truncated_response_start_after():
+    s3 = boto3.client('s3', region_name='us-east-1')
+    s3.create_bucket(Bucket='mybucket')
+    s3.put_object(Bucket='mybucket', Key='one', Body=b'1')
+    s3.put_object(Bucket='mybucket', Key='two', Body=b'22')
+    s3.put_object(Bucket='mybucket', Key='three', Body=b'333')
+
+    # First list
+    resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, StartAfter='one')
+    listed_object = resp['Contents'][0]
+
+    assert listed_object['Key'] == 'three'
+    assert resp['MaxKeys'] == 1
+    assert resp['Prefix'] == ''
+    assert resp['KeyCount'] == 1
+    assert resp['IsTruncated'] == True
+    assert resp['StartAfter'] == 'one'
+    assert 'Delimiter' not in resp
+    assert 'Owner' not in listed_object
+
+    next_token = resp['NextContinuationToken']
+
+    # Second list
+    # The ContinuationToken must take precedence over StartAfter.
+    resp = s3.list_objects_v2(Bucket='mybucket', MaxKeys=1, StartAfter='one',
+            ContinuationToken=next_token)
+    listed_object = resp['Contents'][0]
+
+    assert listed_object['Key'] == 'two'
+    assert resp['MaxKeys'] == 1
+    assert resp['Prefix'] == ''
+    assert resp['KeyCount'] == 1
+    assert resp['IsTruncated'] == False
+    # When ContinuationToken is given, StartAfter is ignored. This also means
+    # AWS does not return it in the response.
+    assert 'StartAfter' not in resp
+    assert 'Delimiter' not in resp
+    assert 'Owner' not in listed_object
+
+
+@mock_s3
+def test_boto3_list_objects_v2_fetch_owner():
+    s3 = boto3.client('s3', region_name='us-east-1')
+    s3.create_bucket(Bucket='mybucket')
+    s3.put_object(Bucket='mybucket', Key='one', Body=b'11')
+
+    resp = s3.list_objects_v2(Bucket='mybucket', FetchOwner=True)
+    owner = resp['Contents'][0]['Owner']
+
+    assert 'ID' in owner
+    assert 'DisplayName' in owner
+    assert len(owner.keys()) == 2
+
 
 @mock_s3
 def test_boto3_bucket_create():