diff --git a/moto/s3/models.py b/moto/s3/models.py index 8d556cb91..03a7d4873 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -14,7 +14,7 @@ UPLOAD_PART_MIN_SIZE = 5242880 class FakeKey(object): - def __init__(self, name, value, storage=None): + def __init__(self, name, value, storage="STANDARD"): self.name = name self.value = value self.last_modified = datetime.datetime.now() @@ -22,7 +22,7 @@ class FakeKey(object): self._metadata = {} self._expiry = None - def copy(self, new_name = None): + def copy(self, new_name=None): r = copy.deepcopy(self) if new_name is not None: r.name = new_name @@ -70,9 +70,8 @@ class FakeKey(object): 'etag': self.etag, 'last-modified': self.last_modified_RFC1123, } - if self._storage_class is not None: - if self._storage_class != 'STANDARD': - r['x-amz-storage-class'] = self._storage_class + if self._storage_class != 'STANDARD': + r['x-amz-storage-class'] = self._storage_class if self._expiry is not None: rhdr = 'ongoing-request="false", expiry-date="{0}"' r['x-amz-restore'] = rhdr.format(self.expiry_date) @@ -84,10 +83,7 @@ class FakeKey(object): @property def storage_class(self): - if self._storage_class is not None: - return self._storage_class - else: - return 'STANDARD' + return self._storage_class @property def expiry_date(self): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 3220ee9c6..d24bfb3ff 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -29,7 +29,6 @@ class ResponseObject(object): response = self._bucket_response(request, full_url, headers) if isinstance(response, basestring): return 200, headers, response - else: status_code, headers, response_content = response return status_code, headers, response_content @@ -45,82 +44,96 @@ class ResponseObject(object): return self.all_buckets() if method == 'HEAD': - bucket = self.backend.get_bucket(bucket_name) - if bucket: - return 200, headers, "" - else: - return 404, headers, "" - + return self._bucket_response_head(bucket_name, headers) elif method == 'GET': - bucket = self.backend.get_bucket(bucket_name) - if bucket: - prefix = querystring.get('prefix', [None])[0] - delimiter = querystring.get('delimiter', [None])[0] - result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter) - template = Template(S3_BUCKET_GET_RESPONSE) - return template.render( - bucket=bucket, - prefix=prefix, - delimiter=delimiter, - result_keys=result_keys, - result_folders=result_folders - ) - else: - return 404, headers, "" + return self._bucket_response_get(bucket_name, querystring, headers) elif method == 'PUT': - try: - new_bucket = self.backend.create_bucket(bucket_name) - except BucketAlreadyExists: - return 409, headers, "" - template = Template(S3_BUCKET_CREATE_RESPONSE) - return template.render(bucket=new_bucket) + return self._bucket_response_put(bucket_name, headers) elif method == 'DELETE': - removed_bucket = self.backend.delete_bucket(bucket_name) - if removed_bucket is None: - # Non-existant bucket - template = Template(S3_DELETE_NON_EXISTING_BUCKET) - return 404, headers, template.render(bucket_name=bucket_name) - elif removed_bucket: - # Bucket exists - template = Template(S3_DELETE_BUCKET_SUCCESS) - return 204, headers, template.render(bucket=removed_bucket) - else: - # Tried to delete a bucket that still has keys - template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) - return 409, headers, template.render(bucket=removed_bucket) + return self._bucket_response_delete(bucket_name, headers) elif method == 'POST': - #POST to bucket-url should create file from form - if hasattr(request, 'form'): - #Not HTTPretty - form = request.form - else: - #HTTPretty, build new form object - form = {} - for kv in request.body.split('&'): - k, v = kv.split('=') - form[k] = v - - key = form['key'] - if 'file' in form: - f = form['file'] - else: - f = request.files['file'].stream.read() - - new_key = self.backend.set_key(bucket_name, key, f) - - #Metadata - meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) - - for form_id in form: - result = meta_regex.match(form_id) - if result: - meta_key = result.group(0).lower() - metadata = form[form_id] - new_key.set_metadata(meta_key, metadata) - return 200, headers, "" + return self._bucket_response_post(request, bucket_name, headers) else: raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + def _bucket_response_head(self, bucket_name, headers): + bucket = self.backend.get_bucket(bucket_name) + if bucket: + return 200, headers, "" + else: + return 404, headers, "" + + def _bucket_response_get(self, bucket_name, querystring, headers): + bucket = self.backend.get_bucket(bucket_name) + if bucket: + prefix = querystring.get('prefix', [None])[0] + delimiter = querystring.get('delimiter', [None])[0] + result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter) + template = Template(S3_BUCKET_GET_RESPONSE) + return template.render( + bucket=bucket, + prefix=prefix, + delimiter=delimiter, + result_keys=result_keys, + result_folders=result_folders + ) + else: + return 404, headers, "" + + def _bucket_response_put(self, bucket_name, headers): + try: + new_bucket = self.backend.create_bucket(bucket_name) + except BucketAlreadyExists: + return 409, headers, "" + template = Template(S3_BUCKET_CREATE_RESPONSE) + return template.render(bucket=new_bucket) + + def _bucket_response_delete(self, bucket_name, headers): + removed_bucket = self.backend.delete_bucket(bucket_name) + if removed_bucket is None: + # Non-existant bucket + template = Template(S3_DELETE_NON_EXISTING_BUCKET) + return 404, headers, template.render(bucket_name=bucket_name) + elif removed_bucket: + # Bucket exists + template = Template(S3_DELETE_BUCKET_SUCCESS) + return 204, headers, template.render(bucket=removed_bucket) + else: + # Tried to delete a bucket that still has keys + template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) + return 409, headers, template.render(bucket=removed_bucket) + + def _bucket_response_post(self, request, bucket_name, headers): + #POST to bucket-url should create file from form + if hasattr(request, 'form'): + #Not HTTPretty + form = request.form + else: + #HTTPretty, build new form object + form = {} + for kv in request.body.split('&'): + k, v = kv.split('=') + form[k] = v + + key = form['key'] + if 'file' in form: + f = form['file'] + else: + f = request.files['file'].stream.read() + + new_key = self.backend.set_key(bucket_name, key, f) + + #Metadata + meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) + + for form_id in form: + result = meta_regex.match(form_id) + if result: + meta_key = result.group(0).lower() + metadata = form[form_id] + new_key.set_metadata(meta_key, metadata) + return 200, headers, "" + def key_response(self, request, full_url, headers): response = self._key_response(request, full_url, headers) if isinstance(response, basestring): @@ -147,7 +160,6 @@ class ResponseObject(object): method = request.method key_name = self.parse_key_name(parsed_url.path) - bucket_name = self.bucket_name_from_url(full_url) if hasattr(request, 'body'): @@ -158,127 +170,142 @@ class ResponseObject(object): body = request.data if method == 'GET': - if 'uploadId' in query: - upload_id = query['uploadId'][0] - parts = self.backend.list_multipart(bucket_name, upload_id) - template = Template(S3_MULTIPART_LIST_RESPONSE) - return 200, headers, template.render( - bucket_name=bucket_name, - key_name=key_name, - upload_id=upload_id, - count=len(parts), - parts=parts - ) - key = self.backend.get_key(bucket_name, key_name) - if key: - headers.update(key.metadata) - return 200, headers, key.value - else: - return 404, headers, "" - if method == 'PUT': - if 'uploadId' in query and 'partNumber' in query: - upload_id = query['uploadId'][0] - part_number = int(query['partNumber'][0]) - if 'x-amz-copy-source' in request.headers: - src = request.headers.get("x-amz-copy-source") - src_bucket, src_key = src.split("/", 1) - key = self.backend.copy_part( - bucket_name, upload_id, part_number, src_bucket, - src_key) - template = Template(S3_MULTIPART_UPLOAD_RESPONSE) - response = template.render(part=key) - else: - key = self.backend.set_part( - bucket_name, upload_id, part_number, body) - response = "" - headers.update(key.response_dict) - return 200, headers, response - - storage_class = request.headers.get('x-amz-storage-class', 'STANDARD') - - if 'x-amz-copy-source' in request.headers: - # Copy key - src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1) - self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, - storage=storage_class) - mdirective = request.headers.get('x-amz-metadata-directive') - if mdirective is not None and mdirective == 'REPLACE': - new_key = self.backend.get_key(bucket_name, key_name) - self._key_set_metadata(request, new_key, replace=True) - template = Template(S3_OBJECT_COPY_RESPONSE) - return template.render(key=src_key) - streaming_request = hasattr(request, 'streaming') and request.streaming - closing_connection = headers.get('connection') == 'close' - if closing_connection and streaming_request: - # Closing the connection of a streaming request. No more data - new_key = self.backend.get_key(bucket_name, key_name) - elif streaming_request: - # Streaming request, more data - new_key = self.backend.append_to_key(bucket_name, key_name, body) - else: - # Initial data - new_key = self.backend.set_key(bucket_name, key_name, body, - storage=storage_class) - request.streaming = True - self._key_set_metadata(request, new_key) - - template = Template(S3_OBJECT_RESPONSE) - headers.update(new_key.response_dict) - return 200, headers, template.render(key=new_key) + return self._key_response_get(bucket_name, query, key_name, headers) + elif method == 'PUT': + return self._key_response_put(request, body, bucket_name, query, key_name, headers) elif method == 'HEAD': - key = self.backend.get_key(bucket_name, key_name) - if key: - headers.update(key.metadata) - headers.update(key.response_dict) - return 200, headers, "" - else: - return 404, headers, "" + return self._key_response_head(bucket_name, key_name, headers) elif method == 'DELETE': - if 'uploadId' in query: - upload_id = query['uploadId'][0] - self.backend.cancel_multipart(bucket_name, upload_id) - return 204, headers, "" - removed_key = self.backend.delete_key(bucket_name, key_name) - template = Template(S3_DELETE_OBJECT_SUCCESS) - return 204, headers, template.render(bucket=removed_key) + return self._key_response_delete(bucket_name, query, key_name, headers) elif method == 'POST': - if body == '' and parsed_url.query == 'uploads': - multipart = self.backend.initiate_multipart(bucket_name, key_name) - template = Template(S3_MULTIPART_INITIATE_RESPONSE) - response = template.render( - bucket_name=bucket_name, - key_name=key_name, - upload_id=multipart.id, - ) - return 200, headers, response - - if 'uploadId' in query: - upload_id = query['uploadId'][0] - key = self.backend.complete_multipart(bucket_name, upload_id) - - if key is not None: - template = Template(S3_MULTIPART_COMPLETE_RESPONSE) - return template.render( - bucket_name=bucket_name, - key_name=key.name, - etag=key.etag, - ) - template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR) - return 400, headers, template.render() - elif parsed_url.query == 'restore': - es = minidom.parseString(body).getElementsByTagName('Days') - days = es[0].childNodes[0].wholeText - key = self.backend.get_key(bucket_name, key_name) - r = 202 - if key.expiry_date is not None: - r = 200 - key.restore(int(days)) - return r, headers, "" - else: - raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far") + return self._key_response_post(body, parsed_url, bucket_name, query, key_name, headers) else: raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + def _key_response_get(self, bucket_name, query, key_name, headers): + if 'uploadId' in query: + upload_id = query['uploadId'][0] + parts = self.backend.list_multipart(bucket_name, upload_id) + template = Template(S3_MULTIPART_LIST_RESPONSE) + return 200, headers, template.render( + bucket_name=bucket_name, + key_name=key_name, + upload_id=upload_id, + count=len(parts), + parts=parts + ) + key = self.backend.get_key(bucket_name, key_name) + if key: + headers.update(key.metadata) + return 200, headers, key.value + else: + return 404, headers, "" + + def _key_response_put(self, request, body, bucket_name, query, key_name, headers): + if 'uploadId' in query and 'partNumber' in query: + upload_id = query['uploadId'][0] + part_number = int(query['partNumber'][0]) + if 'x-amz-copy-source' in request.headers: + src = request.headers.get("x-amz-copy-source") + src_bucket, src_key = src.split("/", 1) + key = self.backend.copy_part( + bucket_name, upload_id, part_number, src_bucket, + src_key) + template = Template(S3_MULTIPART_UPLOAD_RESPONSE) + response = template.render(part=key) + else: + key = self.backend.set_part( + bucket_name, upload_id, part_number, body) + response = "" + headers.update(key.response_dict) + return 200, headers, response + + storage_class = request.headers.get('x-amz-storage-class', 'STANDARD') + + if 'x-amz-copy-source' in request.headers: + # Copy key + src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1) + self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, + storage=storage_class) + mdirective = request.headers.get('x-amz-metadata-directive') + if mdirective is not None and mdirective == 'REPLACE': + new_key = self.backend.get_key(bucket_name, key_name) + self._key_set_metadata(request, new_key, replace=True) + template = Template(S3_OBJECT_COPY_RESPONSE) + return template.render(key=src_key) + streaming_request = hasattr(request, 'streaming') and request.streaming + closing_connection = headers.get('connection') == 'close' + if closing_connection and streaming_request: + # Closing the connection of a streaming request. No more data + new_key = self.backend.get_key(bucket_name, key_name) + elif streaming_request: + # Streaming request, more data + new_key = self.backend.append_to_key(bucket_name, key_name, body) + else: + # Initial data + new_key = self.backend.set_key(bucket_name, key_name, body, + storage=storage_class) + request.streaming = True + self._key_set_metadata(request, new_key) + + template = Template(S3_OBJECT_RESPONSE) + headers.update(new_key.response_dict) + return 200, headers, template.render(key=new_key) + + def _key_response_head(self, bucket_name, key_name, headers): + key = self.backend.get_key(bucket_name, key_name) + if key: + headers.update(key.metadata) + headers.update(key.response_dict) + return 200, headers, "" + else: + return 404, headers, "" + + def _key_response_delete(self, bucket_name, query, key_name, headers): + if 'uploadId' in query: + upload_id = query['uploadId'][0] + self.backend.cancel_multipart(bucket_name, upload_id) + return 204, headers, "" + removed_key = self.backend.delete_key(bucket_name, key_name) + template = Template(S3_DELETE_OBJECT_SUCCESS) + return 204, headers, template.render(bucket=removed_key) + + def _key_response_post(self, body, parsed_url, bucket_name, query, key_name, headers): + if body == '' and parsed_url.query == 'uploads': + multipart = self.backend.initiate_multipart(bucket_name, key_name) + template = Template(S3_MULTIPART_INITIATE_RESPONSE) + response = template.render( + bucket_name=bucket_name, + key_name=key_name, + upload_id=multipart.id, + ) + return 200, headers, response + + if 'uploadId' in query: + upload_id = query['uploadId'][0] + key = self.backend.complete_multipart(bucket_name, upload_id) + + if key is not None: + template = Template(S3_MULTIPART_COMPLETE_RESPONSE) + return template.render( + bucket_name=bucket_name, + key_name=key.name, + etag=key.etag, + ) + template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR) + return 400, headers, template.render() + elif parsed_url.query == 'restore': + es = minidom.parseString(body).getElementsByTagName('Days') + days = es[0].childNodes[0].wholeText + key = self.backend.get_key(bucket_name, key_name) + r = 202 + if key.expiry_date is not None: + r = 200 + key.restore(int(days)) + return r, headers, "" + else: + raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far") + S3ResponseInstance = ResponseObject(s3_backend, bucket_name_from_url, parse_key_name) S3_ALL_BUCKETS = """ diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index e32cd3992..3e8a53917 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -66,6 +66,35 @@ def test_multipart_upload(): bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) +@mock_s3 +def test_multipart_upload_with_copy_key(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "original-key" + key.set_contents_from_string("key_value") + + multipart = bucket.initiate_multipart_upload("the-key") + part1 = '0' * 5242880 + multipart.upload_part_from_file(BytesIO(part1), 1) + multipart.copy_part_from_key("foobar", "original-key", 2) + multipart.complete_upload() + bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + "key_value") + + +@mock_s3 +def test_multipart_upload_cancel(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + multipart = bucket.initiate_multipart_upload("the-key") + part1 = '0' * 5242880 + multipart.upload_part_from_file(BytesIO(part1), 1) + multipart.cancel_upload() + # TODO we really need some sort of assertion here, but we don't currently + # have the ability to list mulipart uploads for a bucket. + + @mock_s3 def test_missing_key(): conn = boto.connect_s3('the_key', 'the_secret') diff --git a/tests/test_s3bucket_path/test_bucket_path_server.py b/tests/test_s3bucket_path/test_bucket_path_server.py index aa2369491..387f6cf0b 100644 --- a/tests/test_s3bucket_path/test_bucket_path_server.py +++ b/tests/test_s3bucket_path/test_bucket_path_server.py @@ -30,6 +30,9 @@ def test_s3_server_bucket_create(): res.status_code.should.equal(200) res.data.should.contain("ListBucketResult") + res = test_client.get('/missing-bucket', 'http://localhost:5000') + res.status_code.should.equal(404) + res = test_client.put('/foobar/bar', 'http://localhost:5000', data='test value') res.status_code.should.equal(200)