S3:list_object_versions() - Implement Delimiter + KeyMarker (#4413)
This commit is contained in:
Bert Blommers
GitHub
parent
230e34748f
commit
d916fd636f
@ -19,7 +19,11 @@ from bisect import insort
|
||||
import pytz
|
||||
|
||||
from moto.core import ACCOUNT_ID, BaseBackend, BaseModel, CloudFormationModel
|
||||
from moto.core.utils import iso_8601_datetime_without_milliseconds_s3, rfc_1123_datetime
|
||||
from moto.core.utils import (
|
||||
iso_8601_datetime_without_milliseconds_s3,
|
||||
rfc_1123_datetime,
|
||||
unix_time_millis,
|
||||
)
|
||||
from moto.cloudwatch.models import MetricDatum
|
||||
from moto.utilities.tagging_service import TaggingService
|
||||
from .exceptions import (
|
||||
@ -1405,23 +1409,6 @@ class S3Backend(BaseBackend):
|
||||
def get_bucket_encryption(self, bucket_name):
|
||||
return self.get_bucket(bucket_name).encryption
|
||||
|
||||
def get_bucket_latest_versions(self, bucket_name):
|
||||
versions = self.list_object_versions(bucket_name)
|
||||
latest_modified_per_key = {}
|
||||
latest_versions = {}
|
||||
|
||||
for version in versions:
|
||||
name = version.name
|
||||
last_modified = version.last_modified
|
||||
version_id = version.version_id
|
||||
latest_modified_per_key[name] = max(
|
||||
last_modified, latest_modified_per_key.get(name, datetime.datetime.min)
|
||||
)
|
||||
if last_modified == latest_modified_per_key[name]:
|
||||
latest_versions[name] = version_id
|
||||
|
||||
return latest_versions
|
||||
|
||||
def list_object_versions(
|
||||
self,
|
||||
bucket_name,
|
||||
@ -1434,14 +1421,44 @@ class S3Backend(BaseBackend):
|
||||
):
|
||||
bucket = self.get_bucket(bucket_name)
|
||||
|
||||
if any((delimiter, key_marker, version_id_marker)):
|
||||
raise NotImplementedError(
|
||||
"Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker"
|
||||
)
|
||||
|
||||
return itertools.chain(
|
||||
*(l for key, l in bucket.keys.iterlists() if key.startswith(prefix))
|
||||
common_prefixes = []
|
||||
requested_versions = []
|
||||
delete_markers = []
|
||||
all_versions = itertools.chain(
|
||||
*(copy.deepcopy(l) for key, l in bucket.keys.iterlists())
|
||||
)
|
||||
all_versions = list(all_versions)
|
||||
# sort by name, revert last-modified-date
|
||||
all_versions.sort(key=lambda r: (r.name, -unix_time_millis(r.last_modified)))
|
||||
last_name = None
|
||||
for version in all_versions:
|
||||
name = version.name
|
||||
# guaranteed to be sorted - so the first key with this name will be the latest
|
||||
version.is_latest = name != last_name
|
||||
if version.is_latest:
|
||||
last_name = name
|
||||
# Differentiate between FakeKey and FakeDeleteMarkers
|
||||
if not isinstance(version, FakeKey):
|
||||
delete_markers.append(version)
|
||||
continue
|
||||
# skip all keys that alphabetically come before keymarker
|
||||
if key_marker and name < key_marker:
|
||||
continue
|
||||
# Filter for keys that start with prefix
|
||||
if not name.startswith(prefix):
|
||||
continue
|
||||
# separate out all keys that contain delimiter
|
||||
if delimiter and delimiter in name:
|
||||
index = name.index(delimiter) + len(delimiter)
|
||||
prefix_including_delimiter = name[0:index]
|
||||
common_prefixes.append(prefix_including_delimiter)
|
||||
continue
|
||||
|
||||
requested_versions.append(version)
|
||||
|
||||
common_prefixes = sorted(set(common_prefixes))
|
||||
|
||||
return requested_versions, common_prefixes, delete_markers
|
||||
|
||||
def get_bucket_policy(self, bucket_name):
|
||||
return self.get_bucket(bucket_name).policy
|
||||
|
||||
@ -11,7 +11,6 @@ from moto.core.utils import (
|
||||
amzn_request_id,
|
||||
str_to_rfc_1123_datetime,
|
||||
py2_strip_unicode_keys,
|
||||
unix_time_millis,
|
||||
)
|
||||
from urllib.parse import (
|
||||
parse_qs,
|
||||
@ -474,7 +473,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
||||
version_id_marker = querystring.get("version-id-marker", [None])[0]
|
||||
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
versions = self.backend.list_object_versions(
|
||||
(
|
||||
versions,
|
||||
common_prefixes,
|
||||
delete_markers,
|
||||
) = self.backend.list_object_versions(
|
||||
bucket_name,
|
||||
delimiter=delimiter,
|
||||
encoding_type=encoding_type,
|
||||
@ -483,30 +486,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
||||
version_id_marker=version_id_marker,
|
||||
prefix=prefix,
|
||||
)
|
||||
latest_versions = self.backend.get_bucket_latest_versions(
|
||||
bucket_name=bucket_name
|
||||
)
|
||||
key_list = []
|
||||
delete_marker_list = []
|
||||
for version in versions:
|
||||
if isinstance(version, FakeKey):
|
||||
key_list.append(version)
|
||||
else:
|
||||
delete_marker_list.append(version)
|
||||
key_list = versions
|
||||
template = self.response_template(S3_BUCKET_GET_VERSIONS)
|
||||
|
||||
key_list.sort(key=lambda r: (r.name, -unix_time_millis(r.last_modified)))
|
||||
return (
|
||||
200,
|
||||
{},
|
||||
template.render(
|
||||
common_prefixes=common_prefixes,
|
||||
key_list=key_list,
|
||||
delete_marker_list=delete_marker_list,
|
||||
latest_versions=latest_versions,
|
||||
delete_marker_list=delete_markers,
|
||||
bucket=bucket,
|
||||
prefix=prefix,
|
||||
max_keys=1000,
|
||||
delimiter="",
|
||||
delimiter=delimiter,
|
||||
key_marker=key_marker,
|
||||
is_truncated="false",
|
||||
),
|
||||
)
|
||||
@ -2243,14 +2237,22 @@ S3_BUCKET_GET_VERSIONS = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
{% if prefix != None %}
|
||||
<Prefix>{{ prefix }}</Prefix>
|
||||
{% endif %}
|
||||
<KeyMarker>{{ key_marker }}</KeyMarker>
|
||||
{% if common_prefixes %}
|
||||
{% for prefix in common_prefixes %}
|
||||
<CommonPrefixes>
|
||||
<Prefix>{{ prefix }}</Prefix>
|
||||
</CommonPrefixes>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<Delimiter>{{ delimiter }}</Delimiter>
|
||||
<KeyMarker>{{ key_marker or "" }}</KeyMarker>
|
||||
<MaxKeys>{{ max_keys }}</MaxKeys>
|
||||
<IsTruncated>{{ is_truncated }}</IsTruncated>
|
||||
{% for key in key_list %}
|
||||
<Version>
|
||||
<Key>{{ key.name }}</Key>
|
||||
<VersionId>{% if key.version_id is none %}null{% else %}{{ key.version_id }}{% endif %}</VersionId>
|
||||
<IsLatest>{% if latest_versions[key.name] == key.version_id %}true{% else %}false{% endif %}</IsLatest>
|
||||
<IsLatest>{{ 'true' if key.is_latest else 'false' }}</IsLatest>
|
||||
<LastModified>{{ key.last_modified_ISO8601 }}</LastModified>
|
||||
<ETag>{{ key.etag }}</ETag>
|
||||
<Size>{{ key.size }}</Size>
|
||||
@ -2265,7 +2267,7 @@ S3_BUCKET_GET_VERSIONS = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<DeleteMarker>
|
||||
<Key>{{ marker.name }}</Key>
|
||||
<VersionId>{{ marker.version_id }}</VersionId>
|
||||
<IsLatest>{% if latest_versions[marker.name] == marker.version_id %}true{% else %}false{% endif %}</IsLatest>
|
||||
<IsLatest>{{ 'true' if marker.is_latest else 'false' }}</IsLatest>
|
||||
<LastModified>{{ marker.last_modified_ISO8601 }}</LastModified>
|
||||
<Owner>
|
||||
<ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>
|
||||
|
||||
@ -36,6 +36,7 @@ from moto import settings, mock_s3, mock_s3_deprecated, mock_config
|
||||
import moto.s3.models as s3model
|
||||
from moto.core.exceptions import InvalidNextTokenException
|
||||
from moto.settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
|
||||
from uuid import uuid4
|
||||
|
||||
if settings.TEST_SERVER_MODE:
|
||||
REDUCED_PART_SIZE = S3_UPLOAD_PART_MIN_SIZE
|
||||
@ -4879,7 +4880,7 @@ def test_boto3_get_object_tagging():
|
||||
@mock_s3
|
||||
def test_boto3_list_object_versions():
|
||||
s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
|
||||
bucket_name = "mybucket"
|
||||
bucket_name = "000" + str(uuid4())
|
||||
key = "key-with-versions"
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
s3.put_bucket_versioning(
|
||||
@ -4902,6 +4903,116 @@ def test_boto3_list_object_versions():
|
||||
response["Body"].read().should.equal(items[-1])
|
||||
|
||||
|
||||
@mock_s3
|
||||
def test_boto3_list_object_versions_with_delimiter():
|
||||
s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
|
||||
bucket_name = "000" + str(uuid4())
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
s3.put_bucket_versioning(
|
||||
Bucket=bucket_name, VersioningConfiguration={"Status": "Enabled"}
|
||||
)
|
||||
for key_index in list(range(1, 5)) + list(range(10, 14)):
|
||||
for version_index in range(1, 4):
|
||||
body = f"data-{version_index}".encode("UTF-8")
|
||||
s3.put_object(
|
||||
Bucket=bucket_name, Key=f"key{key_index}-with-data", Body=body
|
||||
)
|
||||
s3.put_object(
|
||||
Bucket=bucket_name, Key=f"key{key_index}-without-data", Body=b""
|
||||
)
|
||||
response = s3.list_object_versions(Bucket=bucket_name)
|
||||
# All object versions should be returned
|
||||
len(response["Versions"]).should.equal(
|
||||
48
|
||||
) # 8 keys * 2 (one with, one without) * 3 versions per key
|
||||
|
||||
# Use start of key as delimiter
|
||||
response = s3.list_object_versions(Bucket=bucket_name, Delimiter="key1")
|
||||
response.should.have.key("CommonPrefixes").equal([{"Prefix": "key1"}])
|
||||
response.should.have.key("Delimiter").equal("key1")
|
||||
# 3 keys that do not contain the phrase 'key1' (key2, key3, key4) * * 2 * 3
|
||||
response.should.have.key("Versions").length_of(18)
|
||||
|
||||
# Use in-between key as delimiter
|
||||
response = s3.list_object_versions(Bucket=bucket_name, Delimiter="-with-")
|
||||
response.should.have.key("CommonPrefixes").equal(
|
||||
[
|
||||
{"Prefix": "key1-with-"},
|
||||
{"Prefix": "key10-with-"},
|
||||
{"Prefix": "key11-with-"},
|
||||
{"Prefix": "key12-with-"},
|
||||
{"Prefix": "key13-with-"},
|
||||
{"Prefix": "key2-with-"},
|
||||
{"Prefix": "key3-with-"},
|
||||
{"Prefix": "key4-with-"},
|
||||
]
|
||||
)
|
||||
response.should.have.key("Delimiter").equal("-with-")
|
||||
# key(1/10/11/12/13)-without, key(2/3/4)-without
|
||||
response.should.have.key("Versions").length_of(8 * 1 * 3)
|
||||
|
||||
# Use in-between key as delimiter
|
||||
response = s3.list_object_versions(Bucket=bucket_name, Delimiter="1-with-")
|
||||
response.should.have.key("CommonPrefixes").equal(
|
||||
[{"Prefix": "key1-with-"}, {"Prefix": "key11-with-"}]
|
||||
)
|
||||
response.should.have.key("Delimiter").equal("1-with-")
|
||||
response.should.have.key("Versions").length_of(42)
|
||||
all_keys = set([v["Key"] for v in response["Versions"]])
|
||||
all_keys.should.contain("key1-without-data")
|
||||
all_keys.shouldnt.contain("key1-with-data")
|
||||
all_keys.should.contain("key4-with-data")
|
||||
all_keys.should.contain("key4-without-data")
|
||||
|
||||
# Use in-between key as delimiter + prefix
|
||||
response = s3.list_object_versions(
|
||||
Bucket=bucket_name, Prefix="key1", Delimiter="with-"
|
||||
)
|
||||
response.should.have.key("CommonPrefixes").equal(
|
||||
[
|
||||
{"Prefix": "key1-with-"},
|
||||
{"Prefix": "key10-with-"},
|
||||
{"Prefix": "key11-with-"},
|
||||
{"Prefix": "key12-with-"},
|
||||
{"Prefix": "key13-with-"},
|
||||
]
|
||||
)
|
||||
response.should.have.key("Delimiter").equal("with-")
|
||||
response.should.have.key("KeyMarker").equal("")
|
||||
response.shouldnt.have.key("NextKeyMarker")
|
||||
response.should.have.key("Versions").length_of(15)
|
||||
all_keys = set([v["Key"] for v in response["Versions"]])
|
||||
all_keys.should.equal(
|
||||
{
|
||||
"key1-without-data",
|
||||
"key10-without-data",
|
||||
"key11-without-data",
|
||||
"key13-without-data",
|
||||
"key12-without-data",
|
||||
}
|
||||
)
|
||||
|
||||
# Start at KeyMarker, and filter using Prefix+Delimiter for all subsequent keys
|
||||
response = s3.list_object_versions(
|
||||
Bucket=bucket_name, Prefix="key1", Delimiter="with-", KeyMarker="key11"
|
||||
)
|
||||
response.should.have.key("CommonPrefixes").equal(
|
||||
[
|
||||
{"Prefix": "key11-with-"},
|
||||
{"Prefix": "key12-with-"},
|
||||
{"Prefix": "key13-with-"},
|
||||
]
|
||||
)
|
||||
response.should.have.key("Delimiter").equal("with-")
|
||||
response.should.have.key("KeyMarker").equal("key11")
|
||||
response.shouldnt.have.key("NextKeyMarker")
|
||||
response.should.have.key("Versions").length_of(9)
|
||||
all_keys = set([v["Key"] for v in response["Versions"]])
|
||||
all_keys.should.equal(
|
||||
{"key11-without-data", "key12-without-data", "key13-without-data"}
|
||||
)
|
||||
|
||||
|
||||
@mock_s3
|
||||
def test_boto3_list_object_versions_with_versioning_disabled():
|
||||
s3 = boto3.client("s3", region_name=DEFAULT_REGION_NAME)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user