Merge pull request #143 from AsymmetricVentures/bucket_versioning
Added basic implementation of key/bucket versioning
This commit is contained in:
Steve Pulec
commit
50fae3035f
@ -20,3 +20,4 @@ Moto is written by Steve Pulec with contributions from:
|
||||
* [Chris St. Pierre](https://github.com/stpierre)
|
||||
* [Frank Mata](https://github.com/matafc)
|
||||
* [Clint Ecker](https://github.com/clintecker)
|
||||
* [Richard Eames](https://github.com/Naddiseo)
|
||||
|
||||
@ -3,18 +3,20 @@ import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import copy
|
||||
import itertools
|
||||
|
||||
from moto.core import BaseBackend
|
||||
from moto.core.utils import iso_8601_datetime, rfc_1123_datetime
|
||||
from .exceptions import BucketAlreadyExists
|
||||
from .utils import clean_key_name
|
||||
from .utils import clean_key_name, _VersionedKeyStore
|
||||
|
||||
UPLOAD_ID_BYTES = 43
|
||||
UPLOAD_PART_MIN_SIZE = 5242880
|
||||
|
||||
|
||||
class FakeKey(object):
|
||||
def __init__(self, name, value, storage="STANDARD", etag=None):
|
||||
|
||||
def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0):
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.last_modified = datetime.datetime.now()
|
||||
@ -22,6 +24,8 @@ class FakeKey(object):
|
||||
self._metadata = {}
|
||||
self._expiry = None
|
||||
self._etag = etag
|
||||
self._version_id = version_id
|
||||
self._is_versioned = is_versioned
|
||||
|
||||
def copy(self, new_name=None):
|
||||
r = copy.deepcopy(self)
|
||||
@ -42,6 +46,10 @@ class FakeKey(object):
|
||||
self.value += value
|
||||
self.last_modified = datetime.datetime.now()
|
||||
self._etag = None # must recalculate etag
|
||||
if self._is_versioned:
|
||||
self._version_id += 1
|
||||
else:
|
||||
self._is_versioned = 0
|
||||
|
||||
def restore(self, days):
|
||||
self._expiry = datetime.datetime.now() + datetime.timedelta(days)
|
||||
@ -79,6 +87,10 @@ class FakeKey(object):
|
||||
if self._expiry is not None:
|
||||
rhdr = 'ongoing-request="false", expiry-date="{0}"'
|
||||
r['x-amz-restore'] = rhdr.format(self.expiry_date)
|
||||
|
||||
if self._is_versioned:
|
||||
r['x-amz-version-id'] = self._version_id
|
||||
|
||||
return r
|
||||
|
||||
@property
|
||||
@ -137,10 +149,16 @@ class FakeMultipart(object):
|
||||
|
||||
|
||||
class FakeBucket(object):
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.keys = {}
|
||||
self.keys = _VersionedKeyStore()
|
||||
self.multiparts = {}
|
||||
self.versioning_status = None
|
||||
|
||||
@property
|
||||
def is_versioned(self):
|
||||
return self.versioning_status == 'Enabled'
|
||||
|
||||
|
||||
class S3Backend(BaseBackend):
|
||||
@ -171,12 +189,42 @@ class S3Backend(BaseBackend):
|
||||
return self.buckets.pop(bucket_name)
|
||||
return None
|
||||
|
||||
def set_bucket_versioning(self, bucket_name, status):
|
||||
self.buckets[bucket_name].versioning_status = status
|
||||
|
||||
def get_bucket_versioning(self, bucket_name):
|
||||
return self.buckets[bucket_name].versioning_status
|
||||
|
||||
def get_bucket_versions(self, bucket_name, delimiter=None,
|
||||
encoding_type=None,
|
||||
key_marker=None,
|
||||
max_keys=None,
|
||||
version_id_marker=None):
|
||||
bucket = self.buckets[bucket_name]
|
||||
|
||||
if any((delimiter, encoding_type, key_marker, version_id_marker)):
|
||||
raise NotImplementedError(
|
||||
"Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker")
|
||||
|
||||
return itertools.chain(*(l for _, l in bucket.keys.iterlists()))
|
||||
def set_key(self, bucket_name, key_name, value, storage=None, etag=None):
|
||||
key_name = clean_key_name(key_name)
|
||||
|
||||
bucket = self.buckets[bucket_name]
|
||||
new_key = FakeKey(name=key_name, value=value,
|
||||
storage=storage, etag=etag)
|
||||
|
||||
old_key = bucket.keys.get(key_name, None)
|
||||
if old_key is not None and bucket.is_versioned:
|
||||
new_version_id = old_key._version_id + 1
|
||||
else:
|
||||
new_version_id = 0
|
||||
|
||||
new_key = FakeKey(
|
||||
name=key_name,
|
||||
value=value,
|
||||
storage=storage,
|
||||
etag=etag,
|
||||
is_versioned=bucket.is_versioned,
|
||||
version_id=new_version_id)
|
||||
bucket.keys[key_name] = new_key
|
||||
|
||||
return new_key
|
||||
@ -188,11 +236,16 @@ class S3Backend(BaseBackend):
|
||||
key.append_to_value(value)
|
||||
return key
|
||||
|
||||
def get_key(self, bucket_name, key_name):
|
||||
def get_key(self, bucket_name, key_name, version_id=None):
|
||||
key_name = clean_key_name(key_name)
|
||||
bucket = self.get_bucket(bucket_name)
|
||||
if bucket:
|
||||
return bucket.keys.get(key_name)
|
||||
if version_id is None:
|
||||
return bucket.keys.get(key_name)
|
||||
else:
|
||||
for key in bucket.keys.getlist(key_name):
|
||||
if str(key._version_id) == str(version_id):
|
||||
return key
|
||||
|
||||
def initiate_multipart(self, bucket_name, key_name):
|
||||
bucket = self.buckets[bucket_name]
|
||||
|
||||
@ -48,7 +48,7 @@ class ResponseObject(object):
|
||||
elif method == 'GET':
|
||||
return self._bucket_response_get(bucket_name, querystring, headers)
|
||||
elif method == 'PUT':
|
||||
return self._bucket_response_put(bucket_name, headers)
|
||||
return self._bucket_response_put(request, bucket_name, querystring, headers)
|
||||
elif method == 'DELETE':
|
||||
return self._bucket_response_delete(bucket_name, headers)
|
||||
elif method == 'POST':
|
||||
@ -73,6 +73,36 @@ class ResponseObject(object):
|
||||
return 200, headers, template.render(
|
||||
bucket_name=bucket_name,
|
||||
uploads=multiparts)
|
||||
elif 'versioning' in querystring:
|
||||
versioning = self.backend.get_bucket_versioning(bucket_name)
|
||||
template = Template(S3_BUCKET_GET_VERSIONING)
|
||||
return 200, headers, template.render(status=versioning)
|
||||
elif 'versions' in querystring:
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
encoding_type = querystring.get('encoding-type', [None])[0]
|
||||
key_marker = querystring.get('key-marker', [None])[0]
|
||||
max_keys = querystring.get('max-keys', [None])[0]
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
version_id_marker = querystring.get('version-id-marker', [None])[0]
|
||||
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
versions = self.backend.get_bucket_versions(
|
||||
bucket_name,
|
||||
delimiter=delimiter,
|
||||
encoding_type=encoding_type,
|
||||
key_marker=key_marker,
|
||||
max_keys=max_keys,
|
||||
version_id_marker=version_id_marker
|
||||
)
|
||||
template = Template(S3_BUCKET_GET_VERSIONS)
|
||||
return 200, headers, template.render(
|
||||
key_list=versions,
|
||||
bucket=bucket,
|
||||
prefix='',
|
||||
max_keys='',
|
||||
delimiter='',
|
||||
is_truncated='false',
|
||||
)
|
||||
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
if bucket:
|
||||
@ -80,7 +110,7 @@ class ResponseObject(object):
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
|
||||
template = Template(S3_BUCKET_GET_RESPONSE)
|
||||
return template.render(
|
||||
return 200, headers, template.render(
|
||||
bucket=bucket,
|
||||
prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
@ -90,13 +120,22 @@ class ResponseObject(object):
|
||||
else:
|
||||
return 404, headers, ""
|
||||
|
||||
def _bucket_response_put(self, bucket_name, headers):
|
||||
try:
|
||||
new_bucket = self.backend.create_bucket(bucket_name)
|
||||
except BucketAlreadyExists:
|
||||
return 409, headers, ""
|
||||
template = Template(S3_BUCKET_CREATE_RESPONSE)
|
||||
return template.render(bucket=new_bucket)
|
||||
def _bucket_response_put(self, request, bucket_name, querystring, headers):
|
||||
if 'versioning' in querystring:
|
||||
ver = re.search('<Status>([A-Za-z]+)</Status>', request.body)
|
||||
if ver:
|
||||
self.backend.set_bucket_versioning(bucket_name, ver.group(1))
|
||||
template = Template(S3_BUCKET_VERSIONING)
|
||||
return template.render(bucket_versioning_status=ver.group(1))
|
||||
else:
|
||||
return 404, headers, ""
|
||||
else:
|
||||
try:
|
||||
new_bucket = self.backend.create_bucket(bucket_name)
|
||||
except BucketAlreadyExists:
|
||||
return 409, headers, ""
|
||||
template = Template(S3_BUCKET_CREATE_RESPONSE)
|
||||
return 200, headers, template.render(bucket=new_bucket)
|
||||
|
||||
def _bucket_response_delete(self, bucket_name, headers):
|
||||
removed_bucket = self.backend.delete_bucket(bucket_name)
|
||||
@ -224,7 +263,9 @@ class ResponseObject(object):
|
||||
count=len(parts),
|
||||
parts=parts
|
||||
)
|
||||
key = self.backend.get_key(bucket_name, key_name)
|
||||
version_id = query.get('versionId', [None])[0]
|
||||
key = self.backend.get_key(
|
||||
bucket_name, key_name, version_id=version_id)
|
||||
if key:
|
||||
headers.update(key.metadata)
|
||||
return 200, headers, key.value
|
||||
@ -411,6 +452,49 @@ S3_DELETE_BUCKET_WITH_ITEMS_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<HostId>sdfgdsfgdsfgdfsdsfgdfs</HostId>
|
||||
</Error>"""
|
||||
|
||||
S3_BUCKET_VERSIONING = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>{{ bucket_versioning_status }}</Status>
|
||||
</VersioningConfiguration>
|
||||
"""
|
||||
|
||||
S3_BUCKET_GET_VERSIONING = """
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
{% if status is none %}
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>
|
||||
{% else %}
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Status>{{ status }}</Status>
|
||||
</VersioningConfiguration>
|
||||
{% endif %}
|
||||
"""
|
||||
|
||||
S3_BUCKET_GET_VERSIONS = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
|
||||
<Name>{{ bucket.name }}</Name>
|
||||
<Prefix>{{ prefix }}</Prefix>
|
||||
<KeyMarker>{{ key_marker }}</KeyMarker>
|
||||
<MaxKeys>{{ max_keys }}</MaxKeys>
|
||||
<IsTruncated>{{ is_truncated }}</IsTruncated>
|
||||
{% for key in key_list %}
|
||||
<Version>
|
||||
<Key>{{ key.name }}</Key>
|
||||
<VersionId>{{ key._version_id }}</VersionId>
|
||||
<IsLatest>false</IsLatest>
|
||||
<LastModified>{{ key.last_modified_ISO8601 }}</LastModified>
|
||||
<ETag>{{ key.etag }}</ETag>
|
||||
<Size>{{ key.size }}</Size>
|
||||
<StorageClass>{{ key.storage_class }}</StorageClass>
|
||||
<Owner>
|
||||
<ID>75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a</ID>
|
||||
<DisplayName>webfile</DisplayName>
|
||||
</Owner>
|
||||
</Version>
|
||||
{% endfor %}
|
||||
</ListVersionsResult>
|
||||
"""
|
||||
|
||||
S3_DELETE_KEYS_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
|
||||
{% for k in deleted %}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import re
|
||||
import sys
|
||||
import urllib2
|
||||
import urlparse
|
||||
|
||||
@ -25,3 +26,73 @@ def bucket_name_from_url(url):
|
||||
|
||||
def clean_key_name(key_name):
|
||||
return urllib2.unquote(key_name)
|
||||
|
||||
|
||||
class _VersionedKeyStore(dict):
|
||||
|
||||
""" A simplified/modified version of Django's `MultiValueDict` taken from:
|
||||
https://github.com/django/django/blob/70576740b0bb5289873f5a9a9a4e1a26b2c330e5/django/utils/datastructures.py#L282
|
||||
"""
|
||||
|
||||
def __sgetitem__(self, key):
|
||||
return super(_VersionedKeyStore, self).__getitem__(key)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.__sgetitem__(key)[-1]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
try:
|
||||
current = self.__sgetitem__(key)
|
||||
current.append(value)
|
||||
except (KeyError, IndexError):
|
||||
current = [value]
|
||||
|
||||
super(_VersionedKeyStore, self).__setitem__(key, current)
|
||||
|
||||
def get(self, key, default=None):
|
||||
try:
|
||||
return self[key]
|
||||
except (KeyError, IndexError):
|
||||
pass
|
||||
return default
|
||||
|
||||
def getlist(self, key, default=None):
|
||||
try:
|
||||
return self.__sgetitem__(key)
|
||||
except (KeyError, IndexError):
|
||||
pass
|
||||
return default
|
||||
|
||||
def setlist(self, key, list_):
|
||||
if isinstance(list_, tuple):
|
||||
list_ = list(list_)
|
||||
elif not isinstance(list_, list):
|
||||
list_ = [list_]
|
||||
|
||||
super(_VersionedKeyStore, self).__setitem__(key, list_)
|
||||
|
||||
def _iteritems(self):
|
||||
for key in self:
|
||||
yield key, self[key]
|
||||
|
||||
def _itervalues(self):
|
||||
for key in self:
|
||||
yield self[key]
|
||||
|
||||
def _iterlists(self):
|
||||
for key in self:
|
||||
yield key, self.getlist(key)
|
||||
|
||||
items = iteritems = _iteritems
|
||||
lists = iterlists = _iterlists
|
||||
values = itervalues = _itervalues
|
||||
|
||||
if sys.version_info[0] < 3:
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
def values(self):
|
||||
return list(self.itervalues())
|
||||
|
||||
def lists(self):
|
||||
return list(self.iterlists())
|
||||
|
||||
@ -508,3 +508,64 @@ def test_restore_key_headers():
|
||||
key.ongoing_restore.should_not.be.none
|
||||
key.ongoing_restore.should.be.false
|
||||
key.expiry_date.should.equal("Mon, 02 Jan 2012 12:00:00 GMT")
|
||||
|
||||
|
||||
@mock_s3
|
||||
def test_get_versioning_status():
|
||||
conn = boto.connect_s3('the_key', 'the_secret')
|
||||
bucket = conn.create_bucket('foobar')
|
||||
d = bucket.get_versioning_status()
|
||||
d.should.be.empty
|
||||
|
||||
bucket.configure_versioning(versioning=True)
|
||||
d = bucket.get_versioning_status()
|
||||
d.shouldnt.be.empty
|
||||
d.should.have.key('Versioning').being.equal('Enabled')
|
||||
|
||||
bucket.configure_versioning(versioning=False)
|
||||
d = bucket.get_versioning_status()
|
||||
d.should.have.key('Versioning').being.equal('Suspended')
|
||||
|
||||
|
||||
@mock_s3
|
||||
def test_key_version():
|
||||
conn = boto.connect_s3('the_key', 'the_secret')
|
||||
bucket = conn.create_bucket('foobar')
|
||||
bucket.configure_versioning(versioning=True)
|
||||
|
||||
key = Key(bucket)
|
||||
key.key = 'the-key'
|
||||
key.version_id.should.be.none
|
||||
key.set_contents_from_string('some string')
|
||||
key.version_id.should.equal('0')
|
||||
key.set_contents_from_string('some string')
|
||||
key.version_id.should.equal('1')
|
||||
|
||||
key = bucket.get_key('the-key')
|
||||
key.version_id.should.equal('1')
|
||||
|
||||
|
||||
@mock_s3
|
||||
def test_list_versions():
|
||||
conn = boto.connect_s3('the_key', 'the_secret')
|
||||
bucket = conn.create_bucket('foobar')
|
||||
bucket.configure_versioning(versioning=True)
|
||||
|
||||
key = Key(bucket, 'the-key')
|
||||
key.version_id.should.be.none
|
||||
key.set_contents_from_string("Version 1")
|
||||
key.version_id.should.equal('0')
|
||||
key.set_contents_from_string("Version 2")
|
||||
key.version_id.should.equal('1')
|
||||
|
||||
versions = list(bucket.list_versions())
|
||||
|
||||
versions.should.have.length_of(2)
|
||||
|
||||
versions[0].name.should.equal('the-key')
|
||||
versions[0].version_id.should.equal('0')
|
||||
versions[0].get_contents_as_string().should.equal("Version 1")
|
||||
|
||||
versions[1].name.should.equal('the-key')
|
||||
versions[1].version_id.should.equal('1')
|
||||
versions[1].get_contents_as_string().should.equal("Version 2")
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
from sure import expect
|
||||
from moto.s3.utils import bucket_name_from_url
|
||||
from moto.s3.utils import bucket_name_from_url, _VersionedKeyStore
|
||||
|
||||
|
||||
def test_base_url():
|
||||
@ -12,3 +12,40 @@ def test_localhost_bucket():
|
||||
|
||||
def test_localhost_without_bucket():
|
||||
expect(bucket_name_from_url('https://www.localhost:5000/def')).should.equal(None)
|
||||
|
||||
def test_versioned_key_store():
|
||||
d = _VersionedKeyStore()
|
||||
|
||||
d.should.have.length_of(0)
|
||||
|
||||
d['key'] = [1]
|
||||
|
||||
d.should.have.length_of(1)
|
||||
|
||||
d['key'] = 2
|
||||
d.should.have.length_of(1)
|
||||
|
||||
d.should.have.key('key').being.equal(2)
|
||||
|
||||
d.get.when.called_with('key').should.return_value(2)
|
||||
d.get.when.called_with('badkey').should.return_value(None)
|
||||
d.get.when.called_with('badkey', 'HELLO').should.return_value('HELLO')
|
||||
|
||||
# Tests key[
|
||||
d.shouldnt.have.key('badkey')
|
||||
d.__getitem__.when.called_with('badkey').should.throw(KeyError)
|
||||
|
||||
d.getlist('key').should.have.length_of(2)
|
||||
d.getlist('key').should.be.equal([[1], 2])
|
||||
d.getlist('badkey').should.be.none
|
||||
|
||||
d.setlist('key', 1)
|
||||
d.getlist('key').should.be.equal([1])
|
||||
|
||||
d.setlist('key', (1, 2))
|
||||
d.getlist('key').shouldnt.be.equal((1, 2))
|
||||
d.getlist('key').should.be.equal([1, 2])
|
||||
|
||||
d.setlist('key', [[1], [2]])
|
||||
d['key'].should.have.length_of(1)
|
||||
d.getlist('key').should.be.equal([[1], [2]])
|
||||
|
||||
Loading…
Reference in New Issue
Block a user