From 4184acc0d27b584a921939fb85467cebf27bb025 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 2 Apr 2018 14:19:14 -0700 Subject: [PATCH] Added Filtering support for S3 lifecycle (#1535) * Added Filtering support for S3 lifecycle Also added `ExpiredObjectDeleteMarker`. closes #1533 closes #1479 * Result set no longer contains "Prefix" if "Filter" is set. --- moto/s3/models.py | 61 ++++++++++- moto/s3/responses.py | 28 ++++- setup.py | 4 +- tests/test_s3/test_s3_lifecycle.py | 167 ++++++++++++++++++++++++++++- 4 files changed, 253 insertions(+), 7 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index c414225de..3b4623d61 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -15,7 +15,7 @@ from bisect import insort from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \ - InvalidNotificationDestination + InvalidNotificationDestination, MalformedXML from .utils import clean_key_name, _VersionedKeyStore UPLOAD_ID_BYTES = 43 @@ -311,18 +311,35 @@ class FakeTag(BaseModel): self.value = value +class LifecycleFilter(BaseModel): + + def __init__(self, prefix=None, tag=None, and_filter=None): + self.prefix = prefix or '' + self.tag = tag + self.and_filter = and_filter + + +class LifecycleAndFilter(BaseModel): + + def __init__(self, prefix=None, tags=None): + self.prefix = prefix or '' + self.tags = tags + + class LifecycleRule(BaseModel): - def __init__(self, id=None, prefix=None, status=None, expiration_days=None, - expiration_date=None, transition_days=None, + def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None, + expiration_date=None, transition_days=None, expired_object_delete_marker=None, transition_date=None, storage_class=None): self.id = id self.prefix = prefix + self.filter = lc_filter self.status = status self.expiration_days = expiration_days self.expiration_date = expiration_date self.transition_days = transition_days self.transition_date = transition_date + self.expired_object_delete_marker = expired_object_delete_marker self.storage_class = storage_class @@ -387,12 +404,50 @@ class FakeBucket(BaseModel): for rule in rules: expiration = rule.get('Expiration') transition = rule.get('Transition') + + eodm = None + if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: + # This cannot be set if Date or Days is set: + if expiration.get("Days") or expiration.get("Date"): + raise MalformedXML() + eodm = expiration["ExpiredObjectDeleteMarker"] + + # Pull out the filter: + lc_filter = None + if rule.get("Filter"): + # Can't have both `Filter` and `Prefix` (need to check for the presence of the key): + try: + if rule["Prefix"] or not rule["Prefix"]: + raise MalformedXML() + except KeyError: + pass + + and_filter = None + if rule["Filter"].get("And"): + and_tags = [] + if rule["Filter"]["And"].get("Tag"): + if not isinstance(rule["Filter"]["And"]["Tag"], list): + rule["Filter"]["And"]["Tag"] = [rule["Filter"]["And"]["Tag"]] + + for t in rule["Filter"]["And"]["Tag"]: + and_tags.append(FakeTag(t["Key"], t.get("Value", ''))) + + and_filter = LifecycleAndFilter(prefix=rule["Filter"]["And"]["Prefix"], tags=and_tags) + + filter_tag = None + if rule["Filter"].get("Tag"): + filter_tag = FakeTag(rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", '')) + + lc_filter = LifecycleFilter(prefix=rule["Filter"]["Prefix"], tag=filter_tag, and_filter=and_filter) + self.rules.append(LifecycleRule( id=rule.get('ID'), prefix=rule.get('Prefix'), + lc_filter=lc_filter, status=rule['Status'], expiration_days=expiration.get('Days') if expiration else None, expiration_date=expiration.get('Date') if expiration else None, + expired_object_delete_marker=eodm, transition_days=transition.get('Days') if transition else None, transition_date=transition.get('Date') if transition else None, storage_class=transition[ diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 5ae3b0ede..02a9ac40e 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1176,7 +1176,30 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% for rule in rules %} {{ rule.id }} + {% if rule.filter %} + + {{ rule.filter.prefix }} + {% if rule.filter.tag %} + + {{ rule.filter.tag.key }} + {{ rule.filter.tag.value }} + + {% endif %} + {% if rule.filter.and_filter %} + + {{ rule.filter.and_filter.prefix }} + {% for tag in rule.filter.and_filter.tags %} + + {{ tag.key }} + {{ tag.value }} + + {% endfor %} + + {% endif %} + + {% else %} {{ rule.prefix if rule.prefix != None }} + {% endif %} {{ rule.status }} {% if rule.storage_class %} @@ -1189,7 +1212,7 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {{ rule.storage_class }} {% endif %} - {% if rule.expiration_days or rule.expiration_date %} + {% if rule.expiration_days or rule.expiration_date or rule.expired_object_delete_marker %} {% if rule.expiration_days %} {{ rule.expiration_days }} @@ -1197,6 +1220,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% if rule.expiration_date %} {{ rule.expiration_date }} {% endif %} + {% if rule.expired_object_delete_marker %} + {{ rule.expired_object_delete_marker }} + {% endif %} {% endif %} diff --git a/setup.py b/setup.py index f1570c496..1f135ae7b 100755 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.2.1", - "botocore>=1.7.12", + "boto3>=1.6.16", + "botocore>=1.9.16", "cookies", "cryptography>=2.0.0", "requests>=2.5", diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index 5cae8f790..d176e95c6 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -1,12 +1,16 @@ from __future__ import unicode_literals import boto +import boto3 from boto.exception import S3ResponseError from boto.s3.lifecycle import Lifecycle, Transition, Expiration, Rule import sure # noqa +from botocore.exceptions import ClientError +from datetime import datetime +from nose.tools import assert_raises -from moto import mock_s3_deprecated +from moto import mock_s3_deprecated, mock_s3 @mock_s3_deprecated @@ -26,6 +30,167 @@ def test_lifecycle_create(): list(lifecycle.transition).should.equal([]) +@mock_s3 +def test_lifecycle_with_filters(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + # Create a lifecycle rule with a Filter (no tags): + lfc = { + "Rules": [ + { + "Expiration": { + "Days": 7 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["Filter"]["Prefix"] == '' + assert not result["Rules"][0]["Filter"].get("And") + assert not result["Rules"][0]["Filter"].get("Tag") + with assert_raises(KeyError): + assert result["Rules"][0]["Prefix"] + + # With a tag: + lfc["Rules"][0]["Filter"]["Tag"] = { + "Key": "mytag", + "Value": "mytagvalue" + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["Filter"]["Prefix"] == '' + assert not result["Rules"][0]["Filter"].get("And") + assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue" + with assert_raises(KeyError): + assert result["Rules"][0]["Prefix"] + + # With And (single tag): + lfc["Rules"][0]["Filter"]["And"] = { + "Prefix": "some/prefix", + "Tags": [ + { + "Key": "mytag", + "Value": "mytagvalue" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["Filter"]["Prefix"] == "" + assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix" + assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 1 + assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue" + assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue" + with assert_raises(KeyError): + assert result["Rules"][0]["Prefix"] + + # With multiple And tags: + lfc["Rules"][0]["Filter"]["And"] = { + "Prefix": "some/prefix", + "Tags": [ + { + "Key": "mytag", + "Value": "mytagvalue" + }, + { + "Key": "mytag2", + "Value": "mytagvalue2" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["Filter"]["Prefix"] == "" + assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix" + assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 2 + assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue" + assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue" + assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Key"] == "mytag2" + assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Value"] == "mytagvalue2" + assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag" + assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue" + with assert_raises(KeyError): + assert result["Rules"][0]["Prefix"] + + # Can't have both filter and prefix: + lfc["Rules"][0]["Prefix"] = '' + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + lfc["Rules"][0]["Prefix"] = 'some/path' + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + # No filters -- just a prefix: + del lfc["Rules"][0]["Filter"] + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert not result["Rules"][0].get("Filter") + assert result["Rules"][0]["Prefix"] == "some/path" + + +@mock_s3 +def test_lifecycle_with_eodm(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "Expiration": { + "ExpiredObjectDeleteMarker": True + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] + + # Set to False: + lfc["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] = False + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert not result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] + + # With failure: + lfc["Rules"][0]["Expiration"]["Days"] = 7 + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + del lfc["Rules"][0]["Expiration"]["Days"] + + lfc["Rules"][0]["Expiration"]["Date"] = datetime(2015, 1, 1) + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + @mock_s3_deprecated def test_lifecycle_with_glacier_transition(): conn = boto.s3.connect_to_region("us-west-1")