moto/moto/s3/models.py

611 lines
20 KiB
Python
Raw Normal View History

from __future__ import unicode_literals
2013-03-26 14:52:33 +00:00
import os
import base64
2013-03-29 21:45:33 +00:00
import datetime
import hashlib
import copy
2014-06-27 22:21:32 +00:00
import itertools
2014-08-26 17:25:50 +00:00
import codecs
import six
2013-02-18 21:09:40 +00:00
from bisect import insort
2017-03-12 04:41:12 +00:00
from moto.core import BaseBackend, BaseModel
2014-11-30 04:34:40 +00:00
from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
from .exceptions import BucketAlreadyExists, MissingBucket, MissingKey, InvalidPart, EntityTooSmall
from .utils import clean_key_name, _VersionedKeyStore
2013-02-18 21:09:40 +00:00
UPLOAD_ID_BYTES = 43
UPLOAD_PART_MIN_SIZE = 5242880
2013-11-15 09:53:39 +00:00
2013-02-18 21:09:40 +00:00
class FakeDeleteMarker(BaseModel):
def __init__(self, key):
self.key = key
self._version_id = key.version_id + 1
@property
def version_id(self):
return self._version_id
2017-03-12 04:41:12 +00:00
class FakeKey(BaseModel):
def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0):
2013-02-18 21:09:40 +00:00
self.name = name
self.value = value
self.last_modified = datetime.datetime.utcnow()
2015-10-07 07:04:22 +00:00
self.acl = get_canned_acl('private')
self.website_redirect_location = None
2017-02-16 03:35:45 +00:00
self._storage_class = storage if storage else "STANDARD"
self._metadata = {}
2014-03-26 17:15:08 +00:00
self._expiry = None
self._etag = etag
self._version_id = version_id
self._is_versioned = is_versioned
2013-05-17 23:41:39 +00:00
@property
def version_id(self):
return self._version_id
2014-03-30 15:50:36 +00:00
def copy(self, new_name=None):
r = copy.deepcopy(self)
if new_name is not None:
r.name = new_name
return r
def set_metadata(self, metadata, replace=False):
if replace:
self._metadata = {}
self._metadata.update(metadata)
2014-03-26 15:52:31 +00:00
def set_storage_class(self, storage_class):
self._storage_class = storage_class
2015-10-07 07:04:22 +00:00
def set_acl(self, acl):
self.acl = acl
def append_to_value(self, value):
self.value += value
self.last_modified = datetime.datetime.utcnow()
self._etag = None # must recalculate etag
if self._is_versioned:
self._version_id += 1
else:
self._is_versioned = 0
2013-02-18 21:09:40 +00:00
2014-03-26 17:15:08 +00:00
def restore(self, days):
self._expiry = datetime.datetime.utcnow() + datetime.timedelta(days)
2014-03-26 17:15:08 +00:00
2013-02-18 21:09:40 +00:00
@property
def etag(self):
if self._etag is None:
value_md5 = hashlib.md5()
2014-08-26 17:25:50 +00:00
if isinstance(self.value, six.text_type):
value = self.value.encode("utf-8")
else:
value = self.value
value_md5.update(value)
self._etag = value_md5.hexdigest()
return '"{0}"'.format(self._etag)
2013-02-18 21:09:40 +00:00
2013-03-29 21:45:33 +00:00
@property
def last_modified_ISO8601(self):
2014-11-30 04:34:40 +00:00
return iso_8601_datetime_with_milliseconds(self.last_modified)
2013-03-29 21:45:33 +00:00
@property
def last_modified_RFC1123(self):
# Different datetime formats depending on how the key is obtained
# https://github.com/boto/boto/issues/466
2013-05-24 21:22:34 +00:00
return rfc_1123_datetime(self.last_modified)
2013-05-17 23:41:39 +00:00
@property
def metadata(self):
return self._metadata
2013-03-29 21:45:33 +00:00
@property
def response_dict(self):
res = {
'ETag': self.etag,
2013-03-29 21:45:33 +00:00
'last-modified': self.last_modified_RFC1123,
2017-02-16 03:35:45 +00:00
'content-length': str(len(self.value)),
2013-03-29 21:45:33 +00:00
}
2014-03-30 15:50:36 +00:00
if self._storage_class != 'STANDARD':
res['x-amz-storage-class'] = self._storage_class
2014-03-26 17:15:08 +00:00
if self._expiry is not None:
2014-03-27 08:00:50 +00:00
rhdr = 'ongoing-request="false", expiry-date="{0}"'
res['x-amz-restore'] = rhdr.format(self.expiry_date)
if self._is_versioned:
res['x-amz-version-id'] = str(self.version_id)
if self.website_redirect_location:
res['x-amz-website-redirect-location'] = self.website_redirect_location
return res
2013-03-29 21:45:33 +00:00
@property
def size(self):
return len(self.value)
2014-03-26 15:52:31 +00:00
@property
def storage_class(self):
2014-03-30 15:50:36 +00:00
return self._storage_class
2014-03-26 15:52:31 +00:00
2014-03-26 17:15:08 +00:00
@property
def expiry_date(self):
if self._expiry is not None:
return self._expiry.strftime("%a, %d %b %Y %H:%M:%S GMT")
2013-02-26 05:31:01 +00:00
2017-03-12 04:41:12 +00:00
class FakeMultipart(BaseModel):
2017-02-24 02:37:43 +00:00
def __init__(self, key_name, metadata):
2013-03-26 14:52:33 +00:00
self.key_name = key_name
self.metadata = metadata
2013-03-26 14:52:33 +00:00
self.parts = {}
self.partlist = [] # ordered list of part ID's
2014-08-26 17:25:50 +00:00
rand_b64 = base64.b64encode(os.urandom(UPLOAD_ID_BYTES))
self.id = rand_b64.decode('utf-8').replace('=', '').replace('+', '')
def complete(self, body):
2014-08-26 17:25:50 +00:00
decode_hex = codecs.getdecoder("hex_codec")
2013-03-26 14:52:33 +00:00
total = bytearray()
md5s = bytearray()
2013-03-26 14:52:33 +00:00
last = None
count = 0
for pn, etag in body:
part = self.parts.get(pn)
if part is None or part.etag != etag:
raise InvalidPart()
if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE:
raise EntityTooSmall()
2014-08-26 17:25:50 +00:00
part_etag = part.etag.replace('"', '')
md5s.extend(decode_hex(part_etag)[0])
2013-09-30 09:09:35 +00:00
total.extend(part.value)
last = part
count += 1
2013-03-26 14:52:33 +00:00
etag = hashlib.md5()
etag.update(bytes(md5s))
return total, "{0}-{1}".format(etag.hexdigest(), count)
2013-03-26 14:52:33 +00:00
def set_part(self, part_id, value):
if part_id < 1:
return
2013-03-26 14:52:33 +00:00
key = FakeKey(part_id, value)
self.parts[part_id] = key
if part_id not in self.partlist:
insort(self.partlist, part_id)
return key
2013-03-26 14:52:33 +00:00
2013-09-30 09:09:35 +00:00
def list_parts(self):
for part_id in self.partlist:
yield self.parts[part_id]
2013-09-30 09:09:35 +00:00
2013-03-26 14:52:33 +00:00
2017-03-12 04:41:12 +00:00
class FakeGrantee(BaseModel):
2017-02-24 02:37:43 +00:00
2015-10-07 07:04:22 +00:00
def __init__(self, id='', uri='', display_name=''):
self.id = id
self.uri = uri
self.display_name = display_name
@property
def type(self):
return 'Group' if self.uri else 'CanonicalUser'
2017-02-24 02:37:43 +00:00
ALL_USERS_GRANTEE = FakeGrantee(
uri='http://acs.amazonaws.com/groups/global/AllUsers')
AUTHENTICATED_USERS_GRANTEE = FakeGrantee(
uri='http://acs.amazonaws.com/groups/global/AuthenticatedUsers')
LOG_DELIVERY_GRANTEE = FakeGrantee(
uri='http://acs.amazonaws.com/groups/s3/LogDelivery')
2015-10-07 07:04:22 +00:00
PERMISSION_FULL_CONTROL = 'FULL_CONTROL'
PERMISSION_WRITE = 'WRITE'
PERMISSION_READ = 'READ'
PERMISSION_WRITE_ACP = 'WRITE_ACP'
PERMISSION_READ_ACP = 'READ_ACP'
2017-03-12 04:41:12 +00:00
class FakeGrant(BaseModel):
2017-02-24 02:37:43 +00:00
2015-10-07 07:04:22 +00:00
def __init__(self, grantees, permissions):
self.grantees = grantees
self.permissions = permissions
2017-03-12 04:41:12 +00:00
class FakeAcl(BaseModel):
2017-02-24 02:37:43 +00:00
2015-10-07 07:04:22 +00:00
def __init__(self, grants=[]):
self.grants = grants
def get_canned_acl(acl):
2017-02-24 02:37:43 +00:00
owner_grantee = FakeGrantee(
id='75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a')
2015-10-07 07:04:22 +00:00
grants = [FakeGrant([owner_grantee], [PERMISSION_FULL_CONTROL])]
if acl == 'private':
pass # no other permissions
elif acl == 'public-read':
grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ]))
elif acl == 'public-read-write':
2017-02-24 02:37:43 +00:00
grants.append(FakeGrant([ALL_USERS_GRANTEE], [
PERMISSION_READ, PERMISSION_WRITE]))
2015-10-07 07:04:22 +00:00
elif acl == 'authenticated-read':
2017-02-24 02:37:43 +00:00
grants.append(
FakeGrant([AUTHENTICATED_USERS_GRANTEE], [PERMISSION_READ]))
2015-10-07 07:04:22 +00:00
elif acl == 'bucket-owner-read':
pass # TODO: bucket owner ACL
elif acl == 'bucket-owner-full-control':
pass # TODO: bucket owner ACL
elif acl == 'aws-exec-read':
pass # TODO: bucket owner, EC2 Read
2015-10-07 07:04:22 +00:00
elif acl == 'log-delivery-write':
2017-02-24 02:37:43 +00:00
grants.append(FakeGrant([LOG_DELIVERY_GRANTEE], [
PERMISSION_READ_ACP, PERMISSION_WRITE]))
2015-10-07 07:04:22 +00:00
else:
assert False, 'Unknown canned acl: %s' % (acl,)
return FakeAcl(grants=grants)
2017-03-12 04:41:12 +00:00
class LifecycleRule(BaseModel):
2017-02-24 02:37:43 +00:00
2015-06-03 03:11:23 +00:00
def __init__(self, id=None, prefix=None, status=None, expiration_days=None,
expiration_date=None, transition_days=None,
transition_date=None, storage_class=None):
self.id = id
self.prefix = prefix
self.status = status
self.expiration_days = expiration_days
self.expiration_date = expiration_date
self.transition_days = transition_days
self.transition_date = transition_date
self.storage_class = storage_class
2017-03-12 04:41:12 +00:00
class FakeBucket(BaseModel):
2014-12-11 01:44:00 +00:00
def __init__(self, name, region_name):
2013-02-18 21:09:40 +00:00
self.name = name
2014-12-11 01:44:00 +00:00
self.region_name = region_name
self.keys = _VersionedKeyStore()
2013-03-26 14:52:33 +00:00
self.multiparts = {}
self.versioning_status = None
2015-06-03 03:11:23 +00:00
self.rules = []
2015-07-23 21:33:52 +00:00
self.policy = None
self.website_configuration = None
2015-11-12 01:26:29 +00:00
self.acl = get_canned_acl('private')
2014-12-11 01:44:00 +00:00
@property
def location(self):
return self.region_name
@property
def is_versioned(self):
return self.versioning_status == 'Enabled'
2013-02-18 21:09:40 +00:00
2015-06-03 03:11:23 +00:00
def set_lifecycle(self, rules):
self.rules = []
for rule in rules:
expiration = rule.get('Expiration')
transition = rule.get('Transition')
self.rules.append(LifecycleRule(
id=rule.get('ID'),
prefix=rule.get('Prefix'),
2015-06-03 03:11:23 +00:00
status=rule['Status'],
expiration_days=expiration.get('Days') if expiration else None,
expiration_date=expiration.get('Date') if expiration else None,
transition_days=transition.get('Days') if transition else None,
transition_date=transition.get('Date') if transition else None,
2017-02-24 02:37:43 +00:00
storage_class=transition[
'StorageClass'] if transition else None,
2015-06-03 03:11:23 +00:00
))
def delete_lifecycle(self):
self.rules = []
def set_website_configuration(self, website_configuration):
self.website_configuration = website_configuration
def get_cfn_attribute(self, attribute_name):
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
if attribute_name == 'DomainName':
2017-02-24 02:37:43 +00:00
raise NotImplementedError(
'"Fn::GetAtt" : [ "{0}" , "DomainName" ]"')
elif attribute_name == 'WebsiteURL':
2017-02-24 02:37:43 +00:00
raise NotImplementedError(
'"Fn::GetAtt" : [ "{0}" , "WebsiteURL" ]"')
raise UnformattedGetAttTemplateException()
2015-11-12 01:26:29 +00:00
def set_acl(self, acl):
self.acl = acl
2016-08-15 17:57:40 +00:00
@property
def physical_resource_id(self):
return self.name
@classmethod
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name):
bucket = s3_backend.create_bucket(resource_name, region_name)
return bucket
2013-02-18 21:09:40 +00:00
2013-02-18 21:31:42 +00:00
class S3Backend(BaseBackend):
2013-02-18 21:09:40 +00:00
def __init__(self):
self.buckets = {}
2014-12-11 01:44:00 +00:00
def create_bucket(self, bucket_name, region_name):
if bucket_name in self.buckets:
raise BucketAlreadyExists(bucket=bucket_name)
2014-12-11 01:44:00 +00:00
new_bucket = FakeBucket(name=bucket_name, region_name=region_name)
2013-02-18 21:09:40 +00:00
self.buckets[bucket_name] = new_bucket
return new_bucket
2013-02-18 22:31:15 +00:00
def get_all_buckets(self):
return self.buckets.values()
2013-02-18 21:09:40 +00:00
def get_bucket(self, bucket_name):
try:
return self.buckets[bucket_name]
except KeyError:
raise MissingBucket(bucket=bucket_name)
2013-02-18 21:09:40 +00:00
2013-02-18 22:17:19 +00:00
def delete_bucket(self, bucket_name):
bucket = self.get_bucket(bucket_name)
if bucket.keys:
# Can't delete a bucket with keys
return False
else:
return self.buckets.pop(bucket_name)
2013-02-18 22:17:19 +00:00
def set_bucket_versioning(self, bucket_name, status):
self.get_bucket(bucket_name).versioning_status = status
def get_bucket_versioning(self, bucket_name):
return self.get_bucket(bucket_name).versioning_status
def get_bucket_latest_versions(self, bucket_name):
versions = self.get_bucket_versions(bucket_name)
maximum_version_per_key = {}
latest_versions = {}
for version in versions:
if isinstance(version, FakeDeleteMarker):
name = version.key.name
else:
name = version.name
version_id = version.version_id
maximum_version_per_key[name] = max(
version_id,
maximum_version_per_key.get(name, -1)
)
if version_id == maximum_version_per_key[name]:
latest_versions[name] = version_id
return latest_versions
2014-06-27 22:21:32 +00:00
def get_bucket_versions(self, bucket_name, delimiter=None,
encoding_type=None,
key_marker=None,
max_keys=None,
version_id_marker=None):
bucket = self.get_bucket(bucket_name)
2014-06-27 22:21:32 +00:00
if any((delimiter, encoding_type, key_marker, version_id_marker)):
raise NotImplementedError(
"Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker")
return itertools.chain(*(l for _, l in bucket.keys.iterlists()))
2015-07-23 21:33:52 +00:00
def get_bucket_policy(self, bucket_name):
return self.get_bucket(bucket_name).policy
def set_bucket_policy(self, bucket_name, policy):
self.get_bucket(bucket_name).policy = policy
def delete_bucket_policy(self, bucket_name, body):
bucket = self.get_bucket(bucket_name)
bucket.policy = None
2015-06-03 03:11:23 +00:00
def set_bucket_lifecycle(self, bucket_name, rules):
bucket = self.get_bucket(bucket_name)
bucket.set_lifecycle(rules)
def set_bucket_website_configuration(self, bucket_name, website_configuration):
bucket = self.get_bucket(bucket_name)
bucket.set_website_configuration(website_configuration)
def get_bucket_website_configuration(self, bucket_name):
bucket = self.get_bucket(bucket_name)
return bucket.website_configuration
def set_key(self, bucket_name, key_name, value, storage=None, etag=None):
2013-04-13 23:00:37 +00:00
key_name = clean_key_name(key_name)
bucket = self.get_bucket(bucket_name)
old_key = bucket.keys.get(key_name, None)
if old_key is not None and bucket.is_versioned:
new_version_id = old_key._version_id + 1
else:
new_version_id = 0
new_key = FakeKey(
name=key_name,
value=value,
storage=storage,
etag=etag,
is_versioned=bucket.is_versioned,
version_id=new_version_id)
2013-02-18 22:17:19 +00:00
bucket.keys[key_name] = new_key
2013-02-18 21:09:40 +00:00
return new_key
def append_to_key(self, bucket_name, key_name, value):
key_name = clean_key_name(key_name)
key = self.get_key(bucket_name, key_name)
key.append_to_value(value)
return key
2014-06-27 22:21:32 +00:00
def get_key(self, bucket_name, key_name, version_id=None):
2013-04-13 23:00:37 +00:00
key_name = clean_key_name(key_name)
2013-03-05 13:14:43 +00:00
bucket = self.get_bucket(bucket_name)
key = None
2013-03-05 13:14:43 +00:00
if bucket:
2014-06-27 22:21:32 +00:00
if version_id is None:
if key_name in bucket.keys:
key = bucket.keys[key_name]
2014-06-27 22:21:32 +00:00
else:
for key_version in bucket.keys.getlist(key_name):
if str(key_version.version_id) == str(version_id):
key = key_version
break
if isinstance(key, FakeKey):
return key
else:
raise MissingKey(key_name=key_name)
2013-02-18 21:09:40 +00:00
def initiate_multipart(self, bucket_name, key_name, metadata):
bucket = self.get_bucket(bucket_name)
new_multipart = FakeMultipart(key_name, metadata)
2013-03-26 14:52:33 +00:00
bucket.multiparts[new_multipart.id] = new_multipart
return new_multipart
def complete_multipart(self, bucket_name, multipart_id, body):
bucket = self.get_bucket(bucket_name)
2013-03-26 14:52:33 +00:00
multipart = bucket.multiparts[multipart_id]
value, etag = multipart.complete(body)
2013-03-26 14:52:33 +00:00
if value is None:
return
del bucket.multiparts[multipart_id]
2013-03-26 14:52:33 +00:00
key = self.set_key(bucket_name, multipart.key_name, value, etag=etag)
key.set_metadata(multipart.metadata)
return key
2013-03-26 14:52:33 +00:00
2013-09-30 15:36:25 +00:00
def cancel_multipart(self, bucket_name, multipart_id):
bucket = self.get_bucket(bucket_name)
2013-09-30 15:36:25 +00:00
del bucket.multiparts[multipart_id]
2013-09-30 09:09:35 +00:00
def list_multipart(self, bucket_name, multipart_id):
bucket = self.get_bucket(bucket_name)
return list(bucket.multiparts[multipart_id].list_parts())
2013-09-30 09:09:35 +00:00
2014-04-02 16:03:40 +00:00
def get_all_multiparts(self, bucket_name):
bucket = self.get_bucket(bucket_name)
2014-04-02 16:03:40 +00:00
return bucket.multiparts
2013-03-26 14:52:33 +00:00
def set_part(self, bucket_name, multipart_id, part_id, value):
bucket = self.get_bucket(bucket_name)
2013-03-26 14:52:33 +00:00
multipart = bucket.multiparts[multipart_id]
return multipart.set_part(part_id, value)
def copy_part(self, dest_bucket_name, multipart_id, part_id,
src_bucket_name, src_key_name, start_byte, end_byte):
src_key_name = clean_key_name(src_key_name)
src_bucket = self.get_bucket(src_bucket_name)
dest_bucket = self.get_bucket(dest_bucket_name)
multipart = dest_bucket.multiparts[multipart_id]
src_value = src_bucket.keys[src_key_name].value
if start_byte is not None:
src_value = src_value[start_byte:end_byte + 1]
return multipart.set_part(part_id, src_value)
def prefix_query(self, bucket, prefix, delimiter):
key_results = set()
folder_results = set()
if prefix:
2014-08-26 17:25:50 +00:00
for key_name, key in bucket.keys.items():
if key_name.startswith(prefix):
key_without_prefix = key_name.replace(prefix, "", 1)
if delimiter and delimiter in key_without_prefix:
# If delimiter, we need to split out folder_results
2017-02-24 02:37:43 +00:00
key_without_delimiter = key_without_prefix.split(delimiter)[
0]
folder_results.add("{0}{1}{2}".format(
prefix, key_without_delimiter, delimiter))
else:
key_results.add(key)
else:
2014-08-26 17:25:50 +00:00
for key_name, key in bucket.keys.items():
if delimiter and delimiter in key_name:
# If delimiter, we need to split out folder_results
2017-02-24 02:37:43 +00:00
folder_results.add(key_name.split(
delimiter)[0] + delimiter)
else:
key_results.add(key)
key_results = sorted(key_results, key=lambda key: key.name)
2017-02-24 02:37:43 +00:00
folder_results = [folder_name for folder_name in sorted(
folder_results, key=lambda key: key)]
return key_results, folder_results
def _set_delete_marker(self, bucket_name, key_name):
bucket = self.get_bucket(bucket_name)
bucket.keys[key_name] = FakeDeleteMarker(
key=bucket.keys[key_name]
)
def delete_key(self, bucket_name, key_name, version_id=None):
2013-04-13 23:00:37 +00:00
key_name = clean_key_name(key_name)
bucket = self.get_bucket(bucket_name)
try:
if not bucket.is_versioned:
bucket.keys.pop(key_name)
else:
if version_id is None:
self._set_delete_marker(bucket_name, key_name)
else:
if key_name not in bucket.keys:
raise KeyError
bucket.keys.setlist(
key_name,
[
key
for key in bucket.keys.getlist(key_name)
if str(key.version_id) != str(version_id)
]
)
return True
except KeyError:
return False
2013-02-18 21:09:40 +00:00
def copy_key(self, src_bucket_name, src_key_name, dest_bucket_name,
dest_key_name, storage=None, acl=None, src_version_id=None):
2013-04-13 23:00:37 +00:00
src_key_name = clean_key_name(src_key_name)
dest_key_name = clean_key_name(dest_key_name)
dest_bucket = self.get_bucket(dest_bucket_name)
2017-02-24 02:37:43 +00:00
key = self.get_key(src_bucket_name, src_key_name,
version_id=src_version_id)
if dest_key_name != src_key_name:
key = key.copy(dest_key_name)
dest_bucket.keys[dest_key_name] = key
2014-03-26 15:52:31 +00:00
if storage is not None:
2015-10-07 07:04:22 +00:00
key.set_storage_class(storage)
if acl is not None:
key.set_acl(acl)
2013-02-18 21:09:40 +00:00
2015-11-12 01:26:29 +00:00
def set_bucket_acl(self, bucket_name, acl):
bucket = self.get_bucket(bucket_name)
bucket.set_acl(acl)
def get_bucket_acl(self, bucket_name):
bucket = self.get_bucket(bucket_name)
return bucket.acl
2013-02-18 21:31:42 +00:00
s3_backend = S3Backend()