Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use request body in complete upload, improve exception handling

Browse Source
This commit is contained in:
Konstantinos Koukopoulos 2015-02-10 16:56:56 +02:00
parent 8ad39449be
commit 2dd174b577
3 changed files with 63 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
moto/s3/exceptions.py
View File

@ -38,3 +38,36 @@ class MissingBucket(BucketError):
"NoSuchBucket",
"The specified bucket does not exist",
*args, **kwargs)
class InvalidPartOrder(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(InvalidPartOrder, self).__init__(
"InvalidPartOrder",
("The list of parts was not in ascending order. The parts "
"list must be specified in order by part number."),
*args, **kwargs)
class InvalidPart(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(InvalidPart, self).__init__(
"InvalidPart",
("One or more of the specified parts could not be found. "
"The part might not have been uploaded, or the specified "
"entity tag might not have matched the part's entity tag."),
*args, **kwargs)
class EntityTooSmall(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(EntityTooSmall, self).__init__(
"EntityTooSmall",
"Your proposed upload is smaller than the minimum allowed object size.",
*args, **kwargs)

19
moto/s3/models.py
View File

@ -11,7 +11,7 @@ import six
from bisect import insort
from moto.core import BaseBackend
from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
from .exceptions import BucketAlreadyExists, MissingBucket
from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall
from .utils import clean_key_name, _VersionedKeyStore
UPLOAD_ID_BYTES = 43
@ -123,23 +123,28 @@ class FakeMultipart(object):
rand_b64 = base64.b64encode(os.urandom(UPLOAD_ID_BYTES))
self.id = rand_b64.decode('utf-8').replace('=', '').replace('+', '')
def complete(self):
def complete(self, body):
decode_hex = codecs.getdecoder("hex_codec")
total = bytearray()
md5s = bytearray()
last = None
for index, part in enumerate(self.list_parts(), start=1):
count = 0
for pn, etag in body:
part = self.parts.get(pn)
if part is None or part.etag != etag:
raise InvalidPart()
if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE:
return None, None
raise EntityTooSmall()
part_etag = part.etag.replace('"', '')
md5s.extend(decode_hex(part_etag)[0])
total.extend(part.value)
last = part
count += 1
etag = hashlib.md5()
etag.update(bytes(md5s))
return total, "{0}-{1}".format(etag.hexdigest(), index)
return total, "{0}-{1}".format(etag.hexdigest(), count)
def set_part(self, part_id, value):
if part_id < 1:
@ -276,10 +281,10 @@ class S3Backend(BaseBackend):
return new_multipart
def complete_multipart(self, bucket_name, multipart_id):
def complete_multipart(self, bucket_name, multipart_id, body):
bucket = self.get_bucket(bucket_name)
multipart = bucket.multiparts[multipart_id]
value, etag = multipart.complete()
value, etag = multipart.complete(body)
if value is None:
return
del bucket.multiparts[multipart_id]

38
moto/s3/responses.py
View File

@ -7,7 +7,7 @@ from six.moves.urllib.parse import parse_qs, urlparse
from moto.core.responses import _TemplateEnvironmentMixin
from .exceptions import BucketAlreadyExists, S3ClientError
from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder
from .models import s3_backend
from .utils import bucket_name_from_url, metadata_from_headers
from xml.dom import minidom
@ -351,6 +351,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
template = self.response_template(S3_DELETE_OBJECT_SUCCESS)
return 204, headers, template.render(bucket=removed_key)
def _complete_multipart_body(self, body):
ps = minidom.parseString(body).getElementsByTagName('Part')
prev = 0
for p in ps:
pn = int(p.getElementsByTagName('PartNumber')[0].firstChild.wholeText)
if pn <= prev:
raise InvalidPartOrder()
yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText)
def _key_response_post(self, request, body, parsed_url, bucket_name, query, key_name, headers):
if body == b'' and parsed_url.query == 'uploads':
metadata = metadata_from_headers(request.headers)
@ -365,18 +374,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
return 200, headers, response
if 'uploadId' in query:
body = self._complete_multipart_body(body)
upload_id = query['uploadId'][0]
key = self.backend.complete_multipart(bucket_name, upload_id)
if key is not None:
template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
return template.render(
bucket_name=bucket_name,
key_name=key.name,
etag=key.etag,
)
template = self.response_template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
return 400, headers, template.render()
key = self.backend.complete_multipart(bucket_name, upload_id, body)
template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
return template.render(
bucket_name=bucket_name,
key_name=key.name,
etag=key.etag,
)
elif parsed_url.query == 'restore':
es = minidom.parseString(body).getElementsByTagName('Days')
days = es[0].childNodes[0].wholeText
@ -588,14 +594,6 @@ S3_MULTIPART_COMPLETE_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
</CompleteMultipartUploadResult>
"""
S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>EntityTooSmall</Code>
<Message>Your proposed upload is smaller than the minimum allowed object size.</Message>
<RequestId>asdfasdfsdafds</RequestId>
<HostId>sdfgdsfgdsfgdfsdsfgdfs</HostId>
</Error>"""
S3_ALL_MULTIPARTS = """<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>{{ bucket_name }}</Bucket>