Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1852 from jrbeilke/feature-AbortIncompleteMultipartUpload

Browse Source 
Add support for NoncurrentVersionTransition, NoncurrentVersionExpiration, and AbortIncompleteMultipartUpload actions to S3 lifecycle rules
This commit is contained in:
Steve Pulec 2018-10-15 01:00:03 -04:00 committed by GitHub
commit 87e7c7df48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 176 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
moto/s3/models.py
View File

@ -341,8 +341,9 @@ class LifecycleAndFilter(BaseModel):
class LifecycleRule(BaseModel):
def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None,
expiration_date=None, transition_days=None, expired_object_delete_marker=None,
transition_date=None, storage_class=None):
expiration_date=None, transition_days=None, transition_date=None, storage_class=None,
expired_object_delete_marker=None, nve_noncurrent_days=None, nvt_noncurrent_days=None,
nvt_storage_class=None, aimu_days=None):
self.id = id
self.prefix = prefix
self.filter = lc_filter
@ -351,8 +352,12 @@ class LifecycleRule(BaseModel):
self.expiration_date = expiration_date
self.transition_days = transition_days
self.transition_date = transition_date
self.expired_object_delete_marker = expired_object_delete_marker
self.storage_class = storage_class
self.expired_object_delete_marker = expired_object_delete_marker
self.nve_noncurrent_days = nve_noncurrent_days
self.nvt_noncurrent_days = nvt_noncurrent_days
self.nvt_storage_class = nvt_storage_class
self.aimu_days = aimu_days
class CorsRule(BaseModel):
@ -414,9 +419,32 @@ class FakeBucket(BaseModel):
def set_lifecycle(self, rules):
self.rules = []
for rule in rules:
# Extract and validate actions from Lifecycle rule
expiration = rule.get('Expiration')
transition = rule.get('Transition')
nve_noncurrent_days = None
if rule.get('NoncurrentVersionExpiration') is not None:
if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None:
raise MalformedXML()
nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"]
nvt_noncurrent_days = None
nvt_storage_class = None
if rule.get('NoncurrentVersionTransition') is not None:
if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None:
raise MalformedXML()
if rule["NoncurrentVersionTransition"].get('StorageClass') is None:
raise MalformedXML()
nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"]
nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"]
aimu_days = None
if rule.get('AbortIncompleteMultipartUpload') is not None:
if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None:
raise MalformedXML()
aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"]
eodm = None
if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None:
# This cannot be set if Date or Days is set:
@ -459,11 +487,14 @@ class FakeBucket(BaseModel):
status=rule['Status'],
expiration_days=expiration.get('Days') if expiration else None,
expiration_date=expiration.get('Date') if expiration else None,
expired_object_delete_marker=eodm,
transition_days=transition.get('Days') if transition else None,
transition_date=transition.get('Date') if transition else None,
storage_class=transition[
'StorageClass'] if transition else None,
storage_class=transition.get('StorageClass') if transition else None,
expired_object_delete_marker=eodm,
nve_noncurrent_days=nve_noncurrent_days,
nvt_noncurrent_days=nvt_noncurrent_days,
nvt_storage_class=nvt_storage_class,
aimu_days=aimu_days,
))
def delete_lifecycle(self):

16
moto/s3/responses.py
View File

@ -1228,6 +1228,22 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
{% endif %}
</Expiration>
{% endif %}
{% if rule.nvt_noncurrent_days and rule.nvt_storage_class %}
<NoncurrentVersionTransition>
<NoncurrentDays>{{ rule.nvt_noncurrent_days }}</NoncurrentDays>
<StorageClass>{{ rule.nvt_storage_class }}</StorageClass>
</NoncurrentVersionTransition>
{% endif %}
{% if rule.nve_noncurrent_days %}
<NoncurrentVersionExpiration>
<NoncurrentDays>{{ rule.nve_noncurrent_days }}</NoncurrentDays>
</NoncurrentVersionExpiration>
{% endif %}
{% if rule.aimu_days %}
<AbortIncompleteMultipartUpload>
<DaysAfterInitiation>{{ rule.aimu_days }}</DaysAfterInitiation>
</AbortIncompleteMultipartUpload>
{% endif %}
</Rule>
{% endfor %}
</LifecycleConfiguration>

4
tests/test_ec2/test_elastic_block_store.py
View File

@ -615,8 +615,8 @@ def test_copy_snapshot():
dest = dest_ec2.Snapshot(copy_snapshot_response['SnapshotId'])
attribs = ['data_encryption_key_id', 'encrypted',
'kms_key_id', 'owner_alias', 'owner_id', 'progress',
'start_time', 'state', 'state_message',
'kms_key_id', 'owner_alias', 'owner_id',
'progress', 'state', 'state_message',
'tags', 'volume_id', 'volume_size']
for attrib in attribs:

121
tests/test_s3/test_s3_lifecycle.py
View File

@ -191,6 +191,127 @@ def test_lifecycle_with_eodm():
assert err.exception.response["Error"]["Code"] == "MalformedXML"
@mock_s3
def test_lifecycle_with_nve():
client = boto3.client("s3")
client.create_bucket(Bucket="bucket")
lfc = {
"Rules": [
{
"NoncurrentVersionExpiration": {
"NoncurrentDays": 30
},
"ID": "wholebucket",
"Filter": {
"Prefix": ""
},
"Status": "Enabled"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 30
# Change NoncurrentDays:
lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] = 10
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10
# TODO: Add test for failures due to missing children
@mock_s3
def test_lifecycle_with_nvt():
client = boto3.client("s3")
client.create_bucket(Bucket="bucket")
lfc = {
"Rules": [
{
"NoncurrentVersionTransitions": [{
"NoncurrentDays": 30,
"StorageClass": "ONEZONE_IA"
}],
"ID": "wholebucket",
"Filter": {
"Prefix": ""
},
"Status": "Enabled"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 30
assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "ONEZONE_IA"
# Change NoncurrentDays:
lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 10
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 10
# Change StorageClass:
lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] = "GLACIER"
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "GLACIER"
# With failures for missing children:
del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"]
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 30
del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"]
with assert_raises(ClientError) as err:
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
assert err.exception.response["Error"]["Code"] == "MalformedXML"
@mock_s3
def test_lifecycle_with_aimu():
client = boto3.client("s3")
client.create_bucket(Bucket="bucket")
lfc = {
"Rules": [
{
"AbortIncompleteMultipartUpload": {
"DaysAfterInitiation": 7
},
"ID": "wholebucket",
"Filter": {
"Prefix": ""
},
"Status": "Enabled"
}
]
}
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 7
# Change DaysAfterInitiation:
lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] = 30
client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc)
result = client.get_bucket_lifecycle_configuration(Bucket="bucket")
assert len(result["Rules"]) == 1
assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30
# TODO: Add test for failures due to missing children
@mock_s3_deprecated
def test_lifecycle_with_glacier_transition():
conn = boto.s3.connect_to_region("us-west-1")