Added Filtering support for S3 lifecycle (#1535)

* Added Filtering support for S3 lifecycle

Also added `ExpiredObjectDeleteMarker`.

closes #1533
closes #1479

* Result set no longer contains "Prefix" if "Filter" is set.

moto/s3/models.py

@@ -15,7 +15,7 @@ from bisect import insort
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
 from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \
-    InvalidNotificationDestination
+    InvalidNotificationDestination, MalformedXML
 from .utils import clean_key_name, _VersionedKeyStore
 
 UPLOAD_ID_BYTES = 43
@@ -311,18 +311,35 @@ class FakeTag(BaseModel):
         self.value = value
 
 
+class LifecycleFilter(BaseModel):
+
+    def __init__(self, prefix=None, tag=None, and_filter=None):
+        self.prefix = prefix or ''
+        self.tag = tag
+        self.and_filter = and_filter
+
+
+class LifecycleAndFilter(BaseModel):
+
+    def __init__(self, prefix=None, tags=None):
+        self.prefix = prefix or ''
+        self.tags = tags
+
+
 class LifecycleRule(BaseModel):
 
-    def __init__(self, id=None, prefix=None, status=None, expiration_days=None,
+    def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None,
-                 expiration_date=None, transition_days=None,
+                 expiration_date=None, transition_days=None, expired_object_delete_marker=None,
                  transition_date=None, storage_class=None):
         self.id = id
         self.prefix = prefix
+        self.filter = lc_filter
         self.status = status
         self.expiration_days = expiration_days
         self.expiration_date = expiration_date
         self.transition_days = transition_days
         self.transition_date = transition_date
+        self.expired_object_delete_marker = expired_object_delete_marker
         self.storage_class = storage_class
 
 
@@ -387,12 +404,50 @@ class FakeBucket(BaseModel):
         for rule in rules:
             expiration = rule.get('Expiration')
             transition = rule.get('Transition')
+
+            eodm = None
+            if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None:
+                # This cannot be set if Date or Days is set:
+                if expiration.get("Days") or expiration.get("Date"):
+                    raise MalformedXML()
+                eodm = expiration["ExpiredObjectDeleteMarker"]
+
+            # Pull out the filter:
+            lc_filter = None
+            if rule.get("Filter"):
+                # Can't have both `Filter` and `Prefix` (need to check for the presence of the key):
+                try:
+                    if rule["Prefix"] or not rule["Prefix"]:
+                        raise MalformedXML()
+                except KeyError:
+                    pass
+
+                and_filter = None
+                if rule["Filter"].get("And"):
+                    and_tags = []
+                    if rule["Filter"]["And"].get("Tag"):
+                        if not isinstance(rule["Filter"]["And"]["Tag"], list):
+                            rule["Filter"]["And"]["Tag"] = [rule["Filter"]["And"]["Tag"]]
+
+                        for t in rule["Filter"]["And"]["Tag"]:
+                            and_tags.append(FakeTag(t["Key"], t.get("Value", '')))
+
+                    and_filter = LifecycleAndFilter(prefix=rule["Filter"]["And"]["Prefix"], tags=and_tags)
+
+                filter_tag = None
+                if rule["Filter"].get("Tag"):
+                    filter_tag = FakeTag(rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ''))
+
+                lc_filter = LifecycleFilter(prefix=rule["Filter"]["Prefix"], tag=filter_tag, and_filter=and_filter)
+
             self.rules.append(LifecycleRule(
                 id=rule.get('ID'),
                 prefix=rule.get('Prefix'),
+                lc_filter=lc_filter,
                 status=rule['Status'],
                 expiration_days=expiration.get('Days') if expiration else None,
                 expiration_date=expiration.get('Date') if expiration else None,
+                expired_object_delete_marker=eodm,
                 transition_days=transition.get('Days') if transition else None,
                 transition_date=transition.get('Date') if transition else None,
                 storage_class=transition[

moto/s3/responses.py

@@ -1176,7 +1176,30 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
     {% for rule in rules %}
     <Rule>
         <ID>{{ rule.id }}</ID>
+        {% if rule.filter %}
+        <Filter>
+            <Prefix>{{ rule.filter.prefix }}</Prefix>
+            {% if rule.filter.tag %}
+            <Tag>
+                <Key>{{ rule.filter.tag.key }}</Key>
+                <Value>{{ rule.filter.tag.value }}</Value>
+            </Tag>
+            {% endif %}
+            {% if rule.filter.and_filter %}
+            <And>
+                <Prefix>{{ rule.filter.and_filter.prefix }}</Prefix>
+                {% for tag in rule.filter.and_filter.tags %}
+                <Tag>
+                    <Key>{{ tag.key }}</Key>
+                    <Value>{{ tag.value }}</Value>
+                </Tag>
+                {% endfor %}
+            </And>
+            {% endif %}
+        </Filter>
+        {% else %}
         <Prefix>{{ rule.prefix if rule.prefix != None }}</Prefix>
+        {% endif %}
         <Status>{{ rule.status }}</Status>
         {% if rule.storage_class %}
         <Transition>
@@ -1189,7 +1212,7 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
            <StorageClass>{{ rule.storage_class }}</StorageClass>
         </Transition>
         {% endif %}
-        {% if rule.expiration_days or rule.expiration_date %}
+        {% if rule.expiration_days or rule.expiration_date or rule.expired_object_delete_marker %}
         <Expiration>
             {% if rule.expiration_days %}
                <Days>{{ rule.expiration_days }}</Days>
@@ -1197,6 +1220,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
             {% if rule.expiration_date %}
                <Date>{{ rule.expiration_date }}</Date>
             {% endif %}
+            {% if rule.expired_object_delete_marker %}
+                <ExpiredObjectDeleteMarker>{{ rule.expired_object_delete_marker }}</ExpiredObjectDeleteMarker>
+            {% endif %}
         </Expiration>
         {% endif %}
     </Rule>

setup.py

@@ -8,8 +8,8 @@ import sys
 install_requires = [
     "Jinja2>=2.7.3",
     "boto>=2.36.0",
-    "boto3>=1.2.1",
+    "boto3>=1.6.16",
-    "botocore>=1.7.12",
+    "botocore>=1.9.16",
     "cookies",
     "cryptography>=2.0.0",
     "requests>=2.5",

tests/test_s3/test_s3_lifecycle.py

@@ -1,12 +1,16 @@
 from __future__ import unicode_literals
 
 import boto
+import boto3
 from boto.exception import S3ResponseError
 from boto.s3.lifecycle import Lifecycle, Transition, Expiration, Rule
 
 import sure  # noqa
+from botocore.exceptions import ClientError
+from datetime import datetime
+from nose.tools import assert_raises
 
-from moto import mock_s3_deprecated
+from moto import mock_s3_deprecated, mock_s3
 
 
 @mock_s3_deprecated
@@ -26,6 +30,167 @@ def test_lifecycle_create():
     list(lifecycle.transition).should.equal([])
 
 
+@mock_s3
+def test_lifecycle_with_filters():
+    client = boto3.client("s3")
+    client.create_bucket(Bucket="bucket")
+
+    # Create a lifecycle rule with a Filter (no tags):
+    lfc = {
+        "Rules": [
+            {
+                "Expiration": {
+                    "Days": 7
+                },
+                "ID": "wholebucket",
+                "Filter": {
+                    "Prefix": ""
+                },
+                "Status": "Enabled"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ''
+    assert not result["Rules"][0]["Filter"].get("And")
+    assert not result["Rules"][0]["Filter"].get("Tag")
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With a tag:
+    lfc["Rules"][0]["Filter"]["Tag"] = {
+        "Key": "mytag",
+        "Value": "mytagvalue"
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ''
+    assert not result["Rules"][0]["Filter"].get("And")
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With And (single tag):
+    lfc["Rules"][0]["Filter"]["And"] = {
+        "Prefix": "some/prefix",
+        "Tags": [
+            {
+                "Key": "mytag",
+                "Value": "mytagvalue"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ""
+    assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
+    assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 1
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # With multiple And tags:
+    lfc["Rules"][0]["Filter"]["And"] = {
+        "Prefix": "some/prefix",
+        "Tags": [
+            {
+                "Key": "mytag",
+                "Value": "mytagvalue"
+            },
+            {
+                "Key": "mytag2",
+                "Value": "mytagvalue2"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Filter"]["Prefix"] == ""
+    assert result["Rules"][0]["Filter"]["And"]["Prefix"] == "some/prefix"
+    assert len(result["Rules"][0]["Filter"]["And"]["Tags"]) == 2
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][0]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Key"] == "mytag2"
+    assert result["Rules"][0]["Filter"]["And"]["Tags"][1]["Value"] == "mytagvalue2"
+    assert result["Rules"][0]["Filter"]["Tag"]["Key"] == "mytag"
+    assert result["Rules"][0]["Filter"]["Tag"]["Value"] == "mytagvalue"
+    with assert_raises(KeyError):
+        assert result["Rules"][0]["Prefix"]
+
+    # Can't have both filter and prefix:
+    lfc["Rules"][0]["Prefix"] = ''
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+    lfc["Rules"][0]["Prefix"] = 'some/path'
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+    # No filters -- just a prefix:
+    del lfc["Rules"][0]["Filter"]
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert not result["Rules"][0].get("Filter")
+    assert result["Rules"][0]["Prefix"] == "some/path"
+
+
+@mock_s3
+def test_lifecycle_with_eodm():
+    client = boto3.client("s3")
+    client.create_bucket(Bucket="bucket")
+
+    lfc = {
+        "Rules": [
+            {
+                "Expiration": {
+                    "ExpiredObjectDeleteMarker": True
+                },
+                "ID": "wholebucket",
+                "Filter": {
+                    "Prefix": ""
+                },
+                "Status": "Enabled"
+            }
+        ]
+    }
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
+
+    # Set to False:
+    lfc["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"] = False
+    client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
+    assert len(result["Rules"]) == 1
+    assert not result["Rules"][0]["Expiration"]["ExpiredObjectDeleteMarker"]
+
+    # With failure:
+    lfc["Rules"][0]["Expiration"]["Days"] = 7
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+    del lfc["Rules"][0]["Expiration"]["Days"]
+
+    lfc["Rules"][0]["Expiration"]["Date"] = datetime(2015, 1, 1)
+    with assert_raises(ClientError) as err:
+        client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
+    assert err.exception.response["Error"]["Code"] == "MalformedXML"
+
+
 @mock_s3_deprecated
 def test_lifecycle_with_glacier_transition():
     conn = boto.s3.connect_to_region("us-west-1")