Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added Filtering support for S3 lifecycle (#1535)

Browse Source 
* Added Filtering support for S3 lifecycle

Also added `ExpiredObjectDeleteMarker`.

closes #1533
closes #1479

* Result set no longer contains "Prefix" if "Filter" is set.
This commit is contained in:
Mike Grima 2018-04-02 14:19:14 -07:00 committed by Jack Danger
parent 0a4d2037df
commit 4184acc0d2
4 changed files with 253 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
moto/s3/models.py
View File

@ -15,7 +15,7 @@ from bisect import insort
from moto.core import BaseBackend, BaseModel
from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \
InvalidNotificationDestination
InvalidNotificationDestination, MalformedXML
from .utils import clean_key_name, _VersionedKeyStore
UPLOAD_ID_BYTES = 43
@ -311,18 +311,35 @@ class FakeTag(BaseModel):
self.value = value
class LifecycleFilter(BaseModel):
def __init__(self, prefix=None, tag=None, and_filter=None):
self.prefix = prefix or ''
self.tag = tag
self.and_filter = and_filter
class LifecycleAndFilter(BaseModel):
def __init__(self, prefix=None, tags=None):
self.prefix = prefix or ''
self.tags = tags
class LifecycleRule(BaseModel):
def __init__(self, id=None, prefix=None, status=None, expiration_days=None,
expiration_date=None, transition_days=None,
def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None,
expiration_date=None, transition_days=None, expired_object_delete_marker=None,
transition_date=None, storage_class=None):
self.id = id
self.prefix = prefix
self.filter = lc_filter
self.status = status
self.expiration_days = expiration_days
self.expiration_date = expiration_date
self.transition_days = transition_days
self.transition_date = transition_date
self.expired_object_delete_marker = expired_object_delete_marker
self.storage_class = storage_class
@ -387,12 +404,50 @@ class FakeBucket(BaseModel):
for rule in rules:
expiration = rule.get('Expiration')
transition = rule.get('Transition')
eodm = None
if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None:
# This cannot be set if Date or Days is set:
if expiration.get("Days") or expiration.get("Date"):
raise MalformedXML()
eodm = expiration["ExpiredObjectDeleteMarker"]
# Pull out the filter:
lc_filter = None
if rule.get("Filter"):
# Can't have both `Filter` and `Prefix` (need to check for the presence of the key):
try:
if rule["Prefix"] or not rule["Prefix"]:
raise MalformedXML()
except KeyError:
pass
and_filter = None
if rule["Filter"].get("And"):
and_tags = []
if rule["Filter"]["And"].get("Tag"):
if not isinstance(rule["Filter"]["And"]["Tag"], list):
rule["Filter"]["And"]["Tag"] = [rule["Filter"]["And"]["Tag"]]
for t in rule["Filter"]["And"]["Tag"]:
and_tags.append(FakeTag(t["Key"], t.get("Value", '')))
and_filter = LifecycleAndFilter(prefix=rule["Filter"]["And"]["Prefix"], tags=and_tags)
filter_tag = None
if rule["Filter"].get("Tag"):
filter_tag = FakeTag(rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ''))
lc_filter = LifecycleFilter(prefix=rule["Filter"]["Prefix"], tag=filter_tag, and_filter=and_filter)
self.rules.append(LifecycleRule(
id=rule.get('ID'),
prefix=rule.get('Prefix'),
lc_filter=lc_filter,
status=rule['Status'],
expiration_days=expiration.get('Days') if expiration else None,
expiration_date=expiration.get('Date') if expiration else None,
expired_object_delete_marker=eodm,
transition_days=transition.get('Days') if transition else None,
transition_date=transition.get('Date') if transition else None,
storage_class=transition[

28
moto/s3/responses.py
View File

@ -1176,7 +1176,30 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
{% for rule in rules %}
<Rule>
<ID>{{ rule.id }}</ID>
{% if rule.filter %}
<Filter>
<Prefix>{{ rule.filter.prefix }}</Prefix>
{% if rule.filter.tag %}
<Tag>
<Key>{{ rule.filter.tag.key }}</Key>
<Value>{{ rule.filter.tag.value }}</Value>
</Tag>
{% endif %}
{% if rule.filter.and_filter %}
<And>
<Prefix>{{ rule.filter.and_filter.prefix }}</Prefix>
{% for tag in rule.filter.and_filter.tags %}
<Tag>
<Key>{{ tag.key }}</Key>
<Value>{{ tag.value }}</Value>
</Tag>
{% endfor %}
</And>
{% endif %}
</Filter>
{% else %}
<Prefix>{{ rule.prefix if rule.prefix != None }}</Prefix>
{% endif %}
<Status>{{ rule.status }}</Status>
{% if rule.storage_class %}
<Transition>
@ -1189,7 +1212,7 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
<StorageClass>{{ rule.storage_class }}</StorageClass>
</Transition>
{% endif %}
{% if rule.expiration_days or rule.expiration_date %}
{% if rule.expiration_days or rule.expiration_date or rule.expired_object_delete_marker %}
<Expiration>
{% if rule.expiration_days %}
<Days>{{ rule.expiration_days }}</Days>
@ -1197,6 +1220,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
{% if rule.expiration_date %}
<Date>{{ rule.expiration_date }}</Date>
{% endif %}
{% if rule.expired_object_delete_marker %}
<ExpiredObjectDeleteMarker>{{ rule.expired_object_delete_marker }}</ExpiredObjectDeleteMarker>
{% endif %}
</Expiration>
{% endif %}
</Rule>

4
setup.py
View File

@ -8,8 +8,8 @@ import sys
install_requires = [
"Jinja2>=2.7.3",
"boto>=2.36.0",
"boto3>=1.2.1",
"botocore>=1.7.12",
"boto3>=1.6.16",
"botocore>=1.9.16",
"cookies",
"cryptography>=2.0.0",
"requests>=2.5",

167
tests/test_s3/test_s3_lifecycle.py
View File

@ -1,12 +1,16 @@
from __future__ import unicode_literals
import boto
import boto3
from boto.exception import S3ResponseError
from boto.s3.lifecycle import Lifecycle, Transition, Expiration, Rule
import sure # noqa
from botocore.exceptions import ClientError
from datetime import datetime
from nose.tools import assert_raises
from moto import mock_s3_deprecated
from moto import mock_s3_deprecated, mock_s3
@mock_s3_deprecated
@ -26,6 +30,167 @@ def test_lifecycle_create():
list(lifecycle.transition).should.equal([])
@mock_s3
def test_lifecycle_with_filters():
client = boto3.client("s3")
client.create_bucket(Bucket="bucket")
# Create a lifecycle rule with a Filter (no tags):
lfc = {
"Rules": [
{
"Expiration": {
"Days": 7
},
"ID": "wholebucket",
"Filter": {
"Prefix": ""
},
"Status": "Enabled"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["Filter"]["Prefix"] == ''
assert not result["Rules"][0]["Filter"].get("And")
assert not result["Rules"][0]["Filter"].get("Tag")
with assert_raises(KeyError):
assert result["Rules"][0]["Prefix"]
# With a tag:
lfc["Rules"][0]["Filter"]["Tag"] = {
"Key": "mytag",
"Value": "mytagvalue"
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["Filter"]["Prefix"] == ''
assert not result["Rules"][0]["Filter"].get("And")
assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
with assert_raises(KeyError):
assert result["Rules"][0]["Prefix"]
# With And (single tag):
lfc["Rules"][0]["Filter"]["And"] = {
"Prefix": "some/prefix",
"Tags": [
{
"Key": "mytag",
"Value": "mytagvalue"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["Filter"]["Prefix"] == ""
assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 1
assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
with assert_raises(KeyError):
assert result["Rules"][0]["Prefix"]
# With multiple And tags:
lfc["Rules"][0]["Filter"]["And"] = {
"Prefix": "some/prefix",
"Tags": [
{
"Key": "mytag",
"Value": "mytagvalue"
},
{
"Key": "mytag2",
"Value": "mytagvalue2"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["Filter"]["Prefix"] == ""
assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 2
assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Key"] == "mytag2"
assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Value"] == "mytagvalue2"
assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
with assert_raises(KeyError):
assert result["Rules"][0]["Prefix"]
# Can't have both filter and prefix:
lfc["Rules"][0]["Prefix"] = ''
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
lfc["Rules"][0]["Prefix"] = 'some/path'
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
# No filters -- just a prefix:
del lfc["Rules"][0]["Filter"]
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert not result["Rules"][0].get("Filter")
assert result["Rules"][0]["Prefix"] == "some/path"
@mock_s3
def test_lifecycle_with_eodm():
client = boto3.client("s3")
client.create_bucket(Bucket="bucket")
lfc = {
"Rules": [
{
"Expiration": {
"ExpiredObjectDeleteMarker": True
},
"ID": "wholebucket",
"Filter": {
"Prefix": ""
},
"Status": "Enabled"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
# Set to False:
lfc["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] = False
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert not result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
# With failure:
lfc["Rules"][0]["Expiration"]["Days"] = 7
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
del lfc["Rules"][0]["Expiration"]["Days"]
lfc["Rules"][0]["Expiration"]["Date"] = datetime(2015, 1, 1)
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
@mock_s3_deprecated
def test_lifecycle_with_glacier_transition():
conn = boto.s3.connect_to_region("us-west-1")