From f557487e06bde361c953ed719fc3a5ea728455b4 Mon Sep 17 00:00:00 2001 From: Lucian Branescu Mihaila Date: Tue, 26 Mar 2013 14:52:33 +0000 Subject: [PATCH 01/29] Beginning of multipart upload support. --- moto/s3/models.py | 53 ++++++++++++++++++++++++++++++++++++++++ moto/s3/responses.py | 27 ++++++++++++++++++++ tests/test_s3/test_s3.py | 21 ++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/moto/s3/models.py b/moto/s3/models.py index d80eec417..2462d59be 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -1,5 +1,7 @@ # from boto.s3.bucket import Bucket # from boto.s3.key import Key +import os +import base64 import md5 from moto.core import BaseBackend @@ -21,10 +23,40 @@ class FakeKey(object): return len(self.value) +class FakeMultipart(object): + def __init__(self, key_name): + self.key_name = key_name + self.parts = {} + self.id = base64.b64encode(os.urandom(43)).replace('=', '') + + def complete(self): + total = bytearray() + + for part_id, index in enumerate(sorted(self.parts.keys()), start=1): + # Make sure part ids are continuous + if part_id != index: + return + + total.extend(self.parts[part_id]) + + if len(total) < 5242880: + return + + return total + + def set_part(self, part_id, value): + if part_id < 1: + return False + + self.parts[part_id] = value + return True + + class FakeBucket(object): def __init__(self, name): self.name = name self.keys = {} + self.multiparts = {} class S3Backend(BaseBackend): @@ -65,6 +97,27 @@ class S3Backend(BaseBackend): if bucket: return bucket.keys.get(key_name) + def initiate_multipart(self, bucket_name, key_name): + bucket = self.buckets[bucket_name] + new_multipart = FakeMultipart(key_name) + bucket.multiparts[new_multipart.id] = new_multipart + + return new_multipart + + def complete_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + multipart = bucket.multiparts[multipart_id] + value = multipart.complete() + if value is None: + return False + + self.set_key(bucket_name, multipart.key_name, value) + + def set_part(self, bucket_name, multipart_id, part_id, value): + bucket = self.buckets[bucket_name] + multipart = bucket.multiparts[multipart_id] + return multipart.set_part(part_id, value) + def prefix_query(self, bucket, prefix): key_results = set() folder_results = set() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 80a0a9421..370c7cf5b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -106,6 +106,20 @@ def key_response(uri_info, method, body, headers): removed_key = s3_backend.delete_key(bucket_name, key_name) template = Template(S3_DELETE_OBJECT_SUCCESS) return template.render(bucket=removed_key), dict(status=204) + elif method == 'POST': + if body == '' and uri_info.query == 'uploads': + multipart = s3_backend.initiate_multipart(bucket_name, key_name) + template = Template(S3_MULTIPART_RESPONSE) + response = template.render( + bucket_name=bucket_name, + key_name=key_name, + multipart_id=multipart.id, + ) + print response + return response, dict() + else: + import pdb; pdb.set_trace() + raise NotImplementedError("POST is only allowed for multipart uploads") else: raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method)) @@ -202,3 +216,16 @@ S3_OBJECT_COPY_RESPONSE = """ + + {{ bucket_name }} + {{ key_name }} + {{ upload_id }} +""" + +S3_MULTIPART_COMPLETE_RESPONSE = """ +""" + +S3_MULTIPART_ERROR_RESPONSE = """ +""" diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 31e011bfc..1f713fabb 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1,4 +1,5 @@ import urllib2 +from io import BytesIO import boto from boto.exception import S3ResponseError @@ -36,6 +37,26 @@ def test_my_model_save(): conn.get_bucket('mybucket').get_key('steve').get_contents_as_string().should.equal('is awesome') +@mock_s3 +def test_multipart_upload(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + multipart = bucket.initiate_multipart_upload("the-key") + multipart.upload_part_from_file(BytesIO('hello'), 1) + multipart.upload_part_from_file(BytesIO('world'), 1) + # Multipart with total size under 5MB is refused + multipart.complete_upload().should.throw(S3ResponseError) + + multipart = bucket.initiate_multipart_upload("the-key") + part1 = '0' * 5242880 + multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) + part2 = '1' + multipart.upload_part_from_file(BytesIO('1'), 1) + multipart.complete_upload() + bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) + + @mock_s3 def test_missing_key(): conn = boto.connect_s3('the_key', 'the_secret') From 5854219a4c41cef103af63a1a9baf293f58fb519 Mon Sep 17 00:00:00 2001 From: Lucian Branescu Mihaila Date: Tue, 26 Mar 2013 15:50:18 +0000 Subject: [PATCH 02/29] Upload part and complete upload. Somehow, boto doesn't like output I send it, even though it's copy-pasted from its own logs. --- moto/s3/models.py | 14 ++++++++------ moto/s3/responses.py | 41 ++++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 2462d59be..a2547ea47 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -37,7 +37,7 @@ class FakeMultipart(object): if part_id != index: return - total.extend(self.parts[part_id]) + total.extend(self.parts[part_id].value) if len(total) < 5242880: return @@ -46,10 +46,11 @@ class FakeMultipart(object): def set_part(self, part_id, value): if part_id < 1: - return False + return - self.parts[part_id] = value - return True + key = FakeKey(part_id, value) + self.parts[part_id] = key + return key class FakeBucket(object): @@ -109,9 +110,10 @@ class S3Backend(BaseBackend): multipart = bucket.multiparts[multipart_id] value = multipart.complete() if value is None: - return False + return + del bucket.multiparts[multipart_id] - self.set_key(bucket_name, multipart.key_name, value) + return self.set_key(bucket_name, multipart.key_name, value) def set_part(self, bucket_name, multipart_id, part_id, value): bucket = self.buckets[bucket_name] diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 370c7cf5b..fc1fdcec2 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -64,6 +64,7 @@ def key_response(uri_info, method, body, headers): key_name = uri_info.path.lstrip('/') hostname = uri_info.hostname headers = headers_to_dict(headers) + query = parse_qs(uri_info.query) bucket_name = bucket_name_from_hostname(hostname) @@ -74,12 +75,20 @@ def key_response(uri_info, method, body, headers): else: return "", dict(status=404) if method == 'PUT': + if 'uploadId' in query and 'partNumber' in query and body: + upload_id = query['uploadId'][0] + part_number = int(query['partNumber'][0]) + key = s3_backend.set_part(bucket_name, upload_id, part_number, body) + + return '', dict(etag=key.etag) + if 'x-amz-copy-source' in headers: # Copy key src_bucket, src_key = headers.get("x-amz-copy-source").split("/") s3_backend.copy_key(src_bucket, src_key, bucket_name, key_name) template = Template(S3_OBJECT_COPY_RESPONSE) return template.render(key=src_key) + if body is not None: key = s3_backend.get_key(bucket_name, key_name) if not key or body: @@ -107,19 +116,30 @@ def key_response(uri_info, method, body, headers): template = Template(S3_DELETE_OBJECT_SUCCESS) return template.render(bucket=removed_key), dict(status=204) elif method == 'POST': + import pdb; pdb.set_trace() if body == '' and uri_info.query == 'uploads': multipart = s3_backend.initiate_multipart(bucket_name, key_name) - template = Template(S3_MULTIPART_RESPONSE) + template = Template(S3_MULTIPART_INITIATE_RESPONSE) response = template.render( bucket_name=bucket_name, key_name=key_name, multipart_id=multipart.id, ) - print response return response, dict() + + if body == '' and 'uploadId' in query: + upload_id = query['uploadId'][0] + key = s3_backend.complete_multipart(bucket_name, upload_id) + + if key is not None: + template = Template(S3_MULTIPART_COMPLETE_RESPONSE) + return template.render( + bucket_name=bucket_name, + key_name=key.name, + etag=key.etag, + ) else: - import pdb; pdb.set_trace() - raise NotImplementedError("POST is only allowed for multipart uploads") + raise NotImplementedError("Method POST had only been implemented for multipart uploads so far") else: raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method)) @@ -217,15 +237,18 @@ S3_OBJECT_COPY_RESPONSE = """ +S3_MULTIPART_INITIATE_RESPONSE = """ {{ bucket_name }} {{ key_name }} {{ upload_id }} """ -S3_MULTIPART_COMPLETE_RESPONSE = """ -""" - -S3_MULTIPART_ERROR_RESPONSE = """ +S3_MULTIPART_COMPLETE_RESPONSE = """ + + http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} + {{ bucket_name }} + {{ key_name }} + {{ etag }} + """ From 0b45622dcb10847a3b9980123bd9cbee81b39dfb Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 11:50:41 +0300 Subject: [PATCH 03/29] render part upload response correctly --- moto/s3/responses.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index bc359d87d..6b0cdc2d5 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -133,8 +133,8 @@ def _key_response(request, full_url, headers): upload_id = query['uploadId'][0] part_number = int(query['partNumber'][0]) key = s3_backend.set_part(bucket_name, upload_id, part_number, body) - - return '', dict(etag=key.etag) + template = Template(S3_MULTIPART_UPLOAD_RESPONSE) + return 200, headers, template.render(part=key) if 'x-amz-copy-source' in request.headers: # Copy key @@ -310,6 +310,12 @@ S3_MULTIPART_INITIATE_RESPONSE = """ {{ upload_id }} """ +S3_MULTIPART_UPLOAD_RESPONSE = """ + + {{ part.last_modified_ISO8601 }} + {{ part.etag }} +""" + S3_MULTIPART_COMPLETE_RESPONSE = """ http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} From 24ff30f9fc2893ad5aca2d9153e6e20c7b24a836 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 11:51:18 +0300 Subject: [PATCH 04/29] multipart_id doesn't exist, it's upload_id --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6b0cdc2d5..fe37c6f88 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -187,7 +187,7 @@ def _key_response(request, full_url, headers): response = template.render( bucket_name=bucket_name, key_name=key_name, - multipart_id=multipart.id, + upload_id=multipart.id, ) return 200, headers, response From 9746e72e1d17056cd178437bf84b01358f52b5c8 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:09:35 +0300 Subject: [PATCH 05/29] implement list parts --- moto/s3/models.py | 23 +++++++++++++++++------ moto/s3/responses.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 67c48beb4..a5fddda98 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -63,12 +63,8 @@ class FakeMultipart(object): def complete(self): total = bytearray() - for part_id, index in enumerate(sorted(self.parts.keys()), start=1): - # Make sure part ids are continuous - if part_id != index: - return - - total.extend(self.parts[part_id].value) + for part in self.list_parts(): + total.extend(part.value) if len(total) < 5242880: return @@ -83,6 +79,17 @@ class FakeMultipart(object): self.parts[part_id] = key return key + def list_parts(self): + parts = [] + + for part_id, index in enumerate(sorted(self.parts.keys()), start=1): + # Make sure part ids are continuous + if part_id != index: + return + parts.append(self.parts[part_id]) + + return parts + class FakeBucket(object): def __init__(self, name): @@ -156,6 +163,10 @@ class S3Backend(BaseBackend): return self.set_key(bucket_name, multipart.key_name, value) + def list_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + return bucket.multiparts[multipart_id].list_parts() + def set_part(self, bucket_name, multipart_id, part_id, value): bucket = self.buckets[bucket_name] multipart = bucket.multiparts[multipart_id] diff --git a/moto/s3/responses.py b/moto/s3/responses.py index fe37c6f88..a0a6a6989 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -122,6 +122,17 @@ def _key_response(request, full_url, headers): body = request.data if method == 'GET': + if 'uploadId' in query: + upload_id = query['uploadId'][0] + parts = s3_backend.list_multipart(bucket_name, upload_id) + template = Template(S3_MULTIPART_LIST_RESPONSE) + return 200, headers, template.render( + bucket_name=bucket_name, + key_name=key_name, + upload_id=upload_id, + count=len(parts), + parts=parts + ) key = s3_backend.get_key(bucket_name, key_name) if key: headers.update(key.metadata) @@ -316,6 +327,35 @@ S3_MULTIPART_UPLOAD_RESPONSE = """ {{ part.etag }} """ +S3_MULTIPART_LIST_RESPONSE = """ + + {{ bucket_name }} + {{ key_name }} + {{ upload_id }} + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + STANDARD + 1 + {{ count }} + {{ count }} + false + {% for part in parts %} + + {{ part.name }} + {{ part.last_modified_ISO8601 }} + {{ part.etag }} + {{ part.size }} + + {% endfor %} +""" + S3_MULTIPART_COMPLETE_RESPONSE = """ http://{{ bucket_name }}.s3.amazonaws.com/{{ key_name }} From 04789a59fa01be7517ab11fca8d9f9a5e01aad1b Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:10:05 +0300 Subject: [PATCH 06/29] don't forget to send the etag in the headers when uploading a part --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index a0a6a6989..1b9cf8cac 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -145,6 +145,7 @@ def _key_response(request, full_url, headers): part_number = int(query['partNumber'][0]) key = s3_backend.set_part(bucket_name, upload_id, part_number, body) template = Template(S3_MULTIPART_UPLOAD_RESPONSE) + headers.update(key.response_dict) return 200, headers, template.render(part=key) if 'x-amz-copy-source' in request.headers: From 3630b3c21a3631a04b05c51e93881c699691b08b Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 12:10:35 +0300 Subject: [PATCH 07/29] remove plus sign from id because it doesn't get parsed correctly in query string --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index a5fddda98..6244cdd5c 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,7 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '') + self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', ' ') def complete(self): total = bytearray() From aead9bb0d595b0673ce68a39b3cb0971257373fd Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:34:56 +0300 Subject: [PATCH 08/29] Complete MultiPart Operation does specify a body --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 1b9cf8cac..11ad6b4b2 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -203,7 +203,7 @@ def _key_response(request, full_url, headers): ) return 200, headers, response - if body == '' and 'uploadId' in query: + if 'uploadId' in query: upload_id = query['uploadId'][0] key = s3_backend.complete_multipart(bucket_name, upload_id) From 4539012db6a6e0ceb1af958cfa38da0690f2fea6 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:35:24 +0300 Subject: [PATCH 09/29] throw proper error if multipart upload is too small --- moto/s3/responses.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 11ad6b4b2..e85b6a0b4 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -214,6 +214,8 @@ def _key_response(request, full_url, headers): key_name=key.name, etag=key.etag, ) + template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR) + return 400, headers, template.render() else: raise NotImplementedError("Method POST had only been implemented for multipart uploads so far") else: @@ -365,3 +367,11 @@ S3_MULTIPART_COMPLETE_RESPONSE = """ {{ etag }} """ + +S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR = """ + + EntityTooSmall + Your proposed upload is smaller than the minimum allowed object size. + asdfasdfsdafds + sdfgdsfgdsfgdfsdsfgdfs +""" From 8f473554930bc87867532297cdac903cb3ee6594 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 14:50:23 +0300 Subject: [PATCH 10/29] remove plus from id (for real this time) --- moto/s3/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 6244cdd5c..81a745273 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,9 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', ' ') + import pdb; pdb.set_trace() + self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') + def complete(self): total = bytearray() From e49006c72312308f1e5a2a93d2ed0bd243546d06 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 16:36:51 +0300 Subject: [PATCH 11/29] remove pdb --- moto/s3/models.py | 1 - moto/s3/responses.py | 1 - 2 files changed, 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 81a745273..0abc3e4c2 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -58,7 +58,6 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - import pdb; pdb.set_trace() self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') diff --git a/moto/s3/responses.py b/moto/s3/responses.py index e85b6a0b4..ded07e2f0 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -192,7 +192,6 @@ def _key_response(request, full_url, headers): template = Template(S3_DELETE_OBJECT_SUCCESS) return 204, headers, template.render(bucket=removed_key) elif method == 'POST': - import pdb; pdb.set_trace() if body == '' and parsed_url.query == 'uploads': multipart = s3_backend.initiate_multipart(bucket_name, key_name) template = Template(S3_MULTIPART_INITIATE_RESPONSE) From 6e65b5f6d48543ddcce9afc33d32cf1f066ddade Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 16:37:50 +0300 Subject: [PATCH 12/29] fix part numbering in multipart tests --- tests/test_s3/test_s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 5b5fbf564..f4a5ad18d 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -45,7 +45,7 @@ def test_multipart_upload(): multipart = bucket.initiate_multipart_upload("the-key") multipart.upload_part_from_file(BytesIO('hello'), 1) - multipart.upload_part_from_file(BytesIO('world'), 1) + multipart.upload_part_from_file(BytesIO('world'), 2) # Multipart with total size under 5MB is refused multipart.complete_upload().should.throw(S3ResponseError) @@ -53,7 +53,7 @@ def test_multipart_upload(): part1 = '0' * 5242880 multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) part2 = '1' - multipart.upload_part_from_file(BytesIO('1'), 1) + multipart.upload_part_from_file(BytesIO('1'), 2) multipart.complete_upload() bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) From b1d59c7e173f4f13969f232b83059feee4c5a00f Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Mon, 30 Sep 2013 18:36:25 +0300 Subject: [PATCH 13/29] support cancelling of multipart upload --- moto/s3/models.py | 4 ++++ moto/s3/responses.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/moto/s3/models.py b/moto/s3/models.py index 0abc3e4c2..a97e28627 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -164,6 +164,10 @@ class S3Backend(BaseBackend): return self.set_key(bucket_name, multipart.key_name, value) + def cancel_multipart(self, bucket_name, multipart_id): + bucket = self.buckets[bucket_name] + del bucket.multiparts[multipart_id] + def list_multipart(self, bucket_name, multipart_id): bucket = self.buckets[bucket_name] return bucket.multiparts[multipart_id].list_parts() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ded07e2f0..b2ff1b60b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -188,6 +188,10 @@ def _key_response(request, full_url, headers): else: return 404, headers, "" elif method == 'DELETE': + if 'uploadId' in query: + upload_id = query['uploadId'][0] + s3_backend.cancel_multipart(bucket_name, upload_id) + return 204, headers, "" removed_key = s3_backend.delete_key(bucket_name, key_name) template = Template(S3_DELETE_OBJECT_SUCCESS) return 204, headers, template.render(bucket=removed_key) From b64dbcaa12aa284d7a0ae39d9337747522641ee0 Mon Sep 17 00:00:00 2001 From: Mike Attili Date: Thu, 7 Nov 2013 17:07:56 -0500 Subject: [PATCH 14/29] Remove ()'s on complete_upload since should.throw requires a 'callable'. --- tests/test_s3/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f4a5ad18d..8cbd489d0 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -47,7 +47,7 @@ def test_multipart_upload(): multipart.upload_part_from_file(BytesIO('hello'), 1) multipart.upload_part_from_file(BytesIO('world'), 2) # Multipart with total size under 5MB is refused - multipart.complete_upload().should.throw(S3ResponseError) + multipart.complete_upload.should.throw(S3ResponseError) multipart = bucket.initiate_multipart_upload("the-key") part1 = '0' * 5242880 From d9862aaa65aac24d3f66c4e41aebbb55b50bd853 Mon Sep 17 00:00:00 2001 From: Mike Attili Date: Thu, 7 Nov 2013 17:09:53 -0500 Subject: [PATCH 15/29] Correct size check on multipart uploads. All parts except last must be > 5MB --- moto/s3/models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index a97e28627..9ab0a5b8b 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -60,16 +60,15 @@ class FakeMultipart(object): self.parts = {} self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') - def complete(self): total = bytearray() + last_part_name = len(self.list_parts()) for part in self.list_parts(): + if part.name != last_part_name and len(part.value) < 5242880: + return total.extend(part.value) - if len(total) < 5242880: - return - return total def set_part(self, part_id, value): From 5a475881d2b927a43a5e5324285bf7a1776a5858 Mon Sep 17 00:00:00 2001 From: Brock Pytlik Date: Mon, 28 Oct 2013 13:43:25 -0700 Subject: [PATCH 16/29] support bucket names in url paths in s3bucket_path --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/core/models.py | 1 + moto/s3/responses.py | 311 +++++++++--------- moto/s3/urls.py | 6 +- moto/s3bucket_path/__init__.py | 2 + moto/s3bucket_path/models.py | 7 + moto/s3bucket_path/responses.py | 15 + moto/s3bucket_path/urls.py | 20 ++ moto/s3bucket_path/utils.py | 10 + .../test_bucket_path_server.py | 50 +++ .../test_s3bucket_path/test_s3bucket_path.py | 281 ++++++++++++++++ .../test_s3bucket_path_utils.py | 14 + 13 files changed, 566 insertions(+), 154 deletions(-) create mode 100644 moto/s3bucket_path/__init__.py create mode 100644 moto/s3bucket_path/models.py create mode 100644 moto/s3bucket_path/responses.py create mode 100644 moto/s3bucket_path/urls.py create mode 100644 moto/s3bucket_path/utils.py create mode 100644 tests/test_s3bucket_path/test_bucket_path_server.py create mode 100644 tests/test_s3bucket_path/test_s3bucket_path.py create mode 100644 tests/test_s3bucket_path/test_s3bucket_path_utils.py diff --git a/moto/__init__.py b/moto/__init__.py index 57e8eef38..76cc62c55 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -7,6 +7,7 @@ from .ec2 import mock_ec2 from .elb import mock_elb from .emr import mock_emr from .s3 import mock_s3 +from .s3bucket_path import mock_s3bucket_path from .ses import mock_ses from .sqs import mock_sqs from .sts import mock_sts diff --git a/moto/backends.py b/moto/backends.py index 6f375a8f1..0bc766fe3 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -4,6 +4,7 @@ from moto.ec2 import ec2_backend from moto.elb import elb_backend from moto.emr import emr_backend from moto.s3 import s3_backend +from moto.s3bucket_path import s3bucket_path_backend from moto.ses import ses_backend from moto.sqs import sqs_backend from moto.sts import sts_backend @@ -15,6 +16,7 @@ BACKENDS = { 'elb': elb_backend, 'emr': emr_backend, 's3': s3_backend, + 's3bucket_path': s3bucket_path_backend, 'ses': ses_backend, 'sqs': sqs_backend, 'sts': sts_backend, diff --git a/moto/core/models.py b/moto/core/models.py index f3e6ad701..17238fcb0 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -9,6 +9,7 @@ from .utils import convert_regex_to_flask_path class MockAWS(object): def __init__(self, backend): self.backend = backend + HTTPretty.reset() def __call__(self, func): return self.decorate_callable(func) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 2fd6f7dfb..743039920 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -7,173 +7,182 @@ from .models import s3_backend from .utils import bucket_name_from_url -def all_buckets(): - # No bucket specified. Listing all buckets - all_buckets = s3_backend.get_all_buckets() - template = Template(S3_ALL_BUCKETS) - return template.render(buckets=all_buckets) +def parse_key_name(pth): + return pth.lstrip("/") -def bucket_response(request, full_url, headers): - response = _bucket_response(request, full_url, headers) - if isinstance(response, basestring): - return 200, headers, response +class ResponseObject(object): + def __init__(self, backend, bucket_name_from_url, parse_key_name): + self.backend = backend + self.bucket_name_from_url = bucket_name_from_url + self.parse_key_name = parse_key_name - else: - status_code, headers, response_content = response - return status_code, headers, response_content + def all_buckets(self): + # No bucket specified. Listing all buckets + all_buckets = self.backend.get_all_buckets() + template = Template(S3_ALL_BUCKETS) + return template.render(buckets=all_buckets) + def bucket_response(self, request, full_url, headers): + response = self._bucket_response(request, full_url, headers) + if isinstance(response, basestring): + return 200, headers, response -def _bucket_response(request, full_url, headers): - parsed_url = urlparse(full_url) - querystring = parse_qs(parsed_url.query) - method = request.method - - bucket_name = bucket_name_from_url(full_url) - if not bucket_name: - # If no bucket specified, list all buckets - return all_buckets() - - if method == 'GET': - bucket = s3_backend.get_bucket(bucket_name) - if bucket: - prefix = querystring.get('prefix', [None])[0] - delimiter = querystring.get('delimiter', [None])[0] - result_keys, result_folders = s3_backend.prefix_query(bucket, prefix, delimiter) - template = Template(S3_BUCKET_GET_RESPONSE) - return template.render( - bucket=bucket, - prefix=prefix, - delimiter=delimiter, - result_keys=result_keys, - result_folders=result_folders - ) else: - return 404, headers, "" - elif method == 'PUT': - new_bucket = s3_backend.create_bucket(bucket_name) - template = Template(S3_BUCKET_CREATE_RESPONSE) - return template.render(bucket=new_bucket) - elif method == 'DELETE': - removed_bucket = s3_backend.delete_bucket(bucket_name) - if removed_bucket is None: - # Non-existant bucket - template = Template(S3_DELETE_NON_EXISTING_BUCKET) - return 404, headers, template.render(bucket_name=bucket_name) - elif removed_bucket: - # Bucket exists - template = Template(S3_DELETE_BUCKET_SUCCESS) - return 204, headers, template.render(bucket=removed_bucket) - else: - # Tried to delete a bucket that still has keys - template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) - return 409, headers, template.render(bucket=removed_bucket) - elif method == 'POST': - #POST to bucket-url should create file from form - if hasattr(request, 'form'): - #Not HTTPretty - form = request.form - else: - #HTTPretty, build new form object - form = {} - for kv in request.body.split('&'): - k, v = kv.split('=') - form[k] = v + status_code, headers, response_content = response + return status_code, headers, response_content - key = form['key'] - f = form['file'] + def _bucket_response(self, request, full_url, headers): + parsed_url = urlparse(full_url) + querystring = parse_qs(parsed_url.query) + method = request.method - new_key = s3_backend.set_key(bucket_name, key, f) + bucket_name = self.bucket_name_from_url(full_url) + if not bucket_name: + # If no bucket specified, list all buckets + return self.all_buckets() - #Metadata - meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) - for form_id in form: - result = meta_regex.match(form_id) - if result: - meta_key = result.group(0).lower() - metadata = form[form_id] - new_key.set_metadata(meta_key, metadata) - return 200, headers, "" - else: - raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + if method == 'GET': + bucket = self.backend.get_bucket(bucket_name) + if bucket: + prefix = querystring.get('prefix', [None])[0] + delimiter = querystring.get('delimiter', [None])[0] + result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter) + template = Template(S3_BUCKET_GET_RESPONSE) + return template.render( + bucket=bucket, + prefix=prefix, + delimiter=delimiter, + result_keys=result_keys, + result_folders=result_folders + ) + else: + return 404, headers, "" + elif method == 'PUT': + new_bucket = self.backend.create_bucket(bucket_name) + template = Template(S3_BUCKET_CREATE_RESPONSE) + return template.render(bucket=new_bucket) + elif method == 'DELETE': + removed_bucket = self.backend.delete_bucket(bucket_name) + if removed_bucket is None: + # Non-existant bucket + template = Template(S3_DELETE_NON_EXISTING_BUCKET) + return 404, headers, template.render(bucket_name=bucket_name) + elif removed_bucket: + # Bucket exists + template = Template(S3_DELETE_BUCKET_SUCCESS) + return 204, headers, template.render(bucket=removed_bucket) + else: + # Tried to delete a bucket that still has keys + template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) + return 409, headers, template.render(bucket=removed_bucket) + elif method == 'POST': + #POST to bucket-url should create file from form + if hasattr(request, 'form'): + #Not HTTPretty + form = request.form + else: + #HTTPretty, build new form object + form = {} + for kv in request.body.split('&'): + k, v = kv.split('=') + form[k] = v + key = form['key'] + f = form['file'] -def key_response(request, full_url, headers): - response = _key_response(request, full_url, headers) - if isinstance(response, basestring): - return 200, headers, response - else: - status_code, headers, response_content = response - return status_code, headers, response_content - - -def _key_response(request, full_url, headers): - parsed_url = urlparse(full_url) - method = request.method - - key_name = parsed_url.path.lstrip('/') - bucket_name = bucket_name_from_url(full_url) - if hasattr(request, 'body'): - # Boto - body = request.body - else: - # Flask server - body = request.data - - if method == 'GET': - key = s3_backend.get_key(bucket_name, key_name) - if key: - headers.update(key.metadata) - return 200, headers, key.value - else: - return 404, headers, "" - if method == 'PUT': - if 'x-amz-copy-source' in request.headers: - # Copy key - src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/",1) - s3_backend.copy_key(src_bucket, src_key, bucket_name, key_name) - template = Template(S3_OBJECT_COPY_RESPONSE) - return template.render(key=src_key) - streaming_request = hasattr(request, 'streaming') and request.streaming - closing_connection = headers.get('connection') == 'close' - if closing_connection and streaming_request: - # Closing the connection of a streaming request. No more data - new_key = s3_backend.get_key(bucket_name, key_name) - elif streaming_request: - # Streaming request, more data - new_key = s3_backend.append_to_key(bucket_name, key_name, body) - else: - # Initial data - new_key = s3_backend.set_key(bucket_name, key_name, body) - request.streaming = True + new_key = self.backend.set_key(bucket_name, key, f) #Metadata meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) - for header in request.headers: - if isinstance(header, basestring): - result = meta_regex.match(header) - if result: - meta_key = result.group(0).lower() - metadata = request.headers[header] - new_key.set_metadata(meta_key, metadata) - template = Template(S3_OBJECT_RESPONSE) - headers.update(new_key.response_dict) - return 200, headers, template.render(key=new_key) - elif method == 'HEAD': - key = s3_backend.get_key(bucket_name, key_name) - if key: - headers.update(key.metadata) - headers.update(key.response_dict) - return 200, headers, key.value + for form_id in form: + result = meta_regex.match(form_id) + if result: + meta_key = result.group(0).lower() + metadata = form[form_id] + new_key.set_metadata(meta_key, metadata) + return 200, headers, "" else: - return 404, headers, "" - elif method == 'DELETE': - removed_key = s3_backend.delete_key(bucket_name, key_name) - template = Template(S3_DELETE_OBJECT_SUCCESS) - return 204, headers, template.render(bucket=removed_key) - else: - raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + def key_response(self, request, full_url, headers): + response = self._key_response(request, full_url, headers) + if isinstance(response, basestring): + return 200, headers, response + else: + status_code, headers, response_content = response + return status_code, headers, response_content + + def _key_response(self, request, full_url, headers): + parsed_url = urlparse(full_url) + method = request.method + + key_name = self.parse_key_name(parsed_url.path) + + bucket_name = self.bucket_name_from_url(full_url) + + if hasattr(request, 'body'): + # Boto + body = request.body + else: + # Flask server + body = request.data + + if method == 'GET': + key = self.backend.get_key(bucket_name, key_name) + if key: + headers.update(key.metadata) + return 200, headers, key.value + else: + return 404, headers, "" + if method == 'PUT': + if 'x-amz-copy-source' in request.headers: + # Copy key + src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1) + self.backend.copy_key(src_bucket, src_key, bucket_name, key_name) + template = Template(S3_OBJECT_COPY_RESPONSE) + return template.render(key=src_key) + streaming_request = hasattr(request, 'streaming') and request.streaming + closing_connection = headers.get('connection') == 'close' + if closing_connection and streaming_request: + # Closing the connection of a streaming request. No more data + new_key = self.backend.get_key(bucket_name, key_name) + elif streaming_request: + # Streaming request, more data + new_key = self.backend.append_to_key(bucket_name, key_name, body) + else: + # Initial data + new_key = self.backend.set_key(bucket_name, key_name, body) + request.streaming = True + + #Metadata + meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) + for header in request.headers: + if isinstance(header, basestring): + result = meta_regex.match(header) + if result: + meta_key = result.group(0).lower() + metadata = request.headers[header] + new_key.set_metadata(meta_key, metadata) + template = Template(S3_OBJECT_RESPONSE) + headers.update(new_key.response_dict) + return 200, headers, template.render(key=new_key) + elif method == 'HEAD': + key = self.backend.get_key(bucket_name, key_name) + if key: + headers.update(key.metadata) + headers.update(key.response_dict) + return 200, headers, "" + else: + return 404, headers, "" + elif method == 'DELETE': + removed_key = self.backend.delete_key(bucket_name, key_name) + template = Template(S3_DELETE_OBJECT_SUCCESS) + return 204, headers, template.render(bucket=removed_key) + else: + raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method)) + +S3ResponseInstance = ResponseObject(s3_backend, bucket_name_from_url, parse_key_name) S3_ALL_BUCKETS = """ diff --git a/moto/s3/urls.py b/moto/s3/urls.py index 21370c15a..5f9bc0cf1 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -1,10 +1,10 @@ -from .responses import bucket_response, key_response +from .responses import S3ResponseInstance url_bases = [ "https?://(?P[a-zA-Z0-9\-_.]*)\.?s3.amazonaws.com" ] url_paths = { - '{0}/$': bucket_response, - '{0}/(?P[a-zA-Z0-9\-_.]+)': key_response, + '{0}/$': S3ResponseInstance.bucket_response, + '{0}/(?P[a-zA-Z0-9\-_.]+)': S3ResponseInstance.key_response, } diff --git a/moto/s3bucket_path/__init__.py b/moto/s3bucket_path/__init__.py new file mode 100644 index 000000000..6dd680bed --- /dev/null +++ b/moto/s3bucket_path/__init__.py @@ -0,0 +1,2 @@ +from .models import s3bucket_path_backend +mock_s3bucket_path = s3bucket_path_backend.decorator diff --git a/moto/s3bucket_path/models.py b/moto/s3bucket_path/models.py new file mode 100644 index 000000000..2b7e99539 --- /dev/null +++ b/moto/s3bucket_path/models.py @@ -0,0 +1,7 @@ +from moto.s3.models import S3Backend + + +class S3BucketPathBackend(S3Backend): + True + +s3bucket_path_backend = S3BucketPathBackend() diff --git a/moto/s3bucket_path/responses.py b/moto/s3bucket_path/responses.py new file mode 100644 index 000000000..0f54a1a1d --- /dev/null +++ b/moto/s3bucket_path/responses.py @@ -0,0 +1,15 @@ +from .models import s3bucket_path_backend + +from .utils import bucket_name_from_url + +from moto.s3.responses import ResponseObject + + +def parse_key_name(pth): + return "/".join(pth.rstrip("/").split("/")[2:]) + +S3BucketPathResponseInstance = ResponseObject( + s3bucket_path_backend, + bucket_name_from_url, + parse_key_name, +) diff --git a/moto/s3bucket_path/urls.py b/moto/s3bucket_path/urls.py new file mode 100644 index 000000000..28f1debc8 --- /dev/null +++ b/moto/s3bucket_path/urls.py @@ -0,0 +1,20 @@ +from .responses import S3BucketPathResponseInstance as ro + +url_bases = [ + "https?://s3.amazonaws.com" +] + + +def bucket_response2(*args): + return ro.bucket_response(*args) + + +def bucket_response3(*args): + return ro.bucket_response(*args) + +url_paths = { + '{0}/$': bucket_response3, + '{0}/(?P[a-zA-Z0-9\-_.]+)$': ro.bucket_response, + '{0}/(?P[a-zA-Z0-9\-_.]+)/$': bucket_response2, + '{0}/(?P[a-zA-Z0-9\-_./]+)/(?P[a-zA-Z0-9\-_.?]+)': ro.key_response +} diff --git a/moto/s3bucket_path/utils.py b/moto/s3bucket_path/utils.py new file mode 100644 index 000000000..97f1d40f1 --- /dev/null +++ b/moto/s3bucket_path/utils.py @@ -0,0 +1,10 @@ +import urlparse + + +def bucket_name_from_url(url): + pth = urlparse.urlparse(url).path.lstrip("/") + + l = pth.lstrip("/").split("/") + if len(l) == 0 or l[0] == "": + return None + return l[0] diff --git a/tests/test_s3bucket_path/test_bucket_path_server.py b/tests/test_s3bucket_path/test_bucket_path_server.py new file mode 100644 index 000000000..943615767 --- /dev/null +++ b/tests/test_s3bucket_path/test_bucket_path_server.py @@ -0,0 +1,50 @@ +import sure # noqa + +import moto.server as server + +''' +Test the different server responses +''' +server.configure_urls("s3bucket_path") + + +def test_s3_server_get(): + test_client = server.app.test_client() + res = test_client.get('/') + + res.data.should.contain('ListAllMyBucketsResult') + + +def test_s3_server_bucket_create(): + test_client = server.app.test_client() + res = test_client.put('/foobar', 'http://localhost:5000') + res.status_code.should.equal(200) + + res = test_client.get('/') + res.data.should.contain('foobar') + + res = test_client.get('/foobar', 'http://localhost:5000') + res.status_code.should.equal(200) + res.data.should.contain("ListBucketResult") + + res = test_client.put('/foobar/bar', 'http://localhost:5000', data='test value') + res.status_code.should.equal(200) + + res = test_client.get('/foobar/bar', 'http://localhost:5000') + res.status_code.should.equal(200) + res.data.should.equal("test value") + + +def test_s3_server_post_to_bucket(): + test_client = server.app.test_client() + res = test_client.put('/foobar', 'http://localhost:5000/') + res.status_code.should.equal(200) + + test_client.post('/foobar', "https://localhost:5000/", data={ + 'key': 'the-key', + 'file': 'nothing' + }) + + res = test_client.get('/foobar/the-key', 'http://localhost:5000/') + res.status_code.should.equal(200) + res.data.should.equal("nothing") diff --git a/tests/test_s3bucket_path/test_s3bucket_path.py b/tests/test_s3bucket_path/test_s3bucket_path.py new file mode 100644 index 000000000..1f62f23eb --- /dev/null +++ b/tests/test_s3bucket_path/test_s3bucket_path.py @@ -0,0 +1,281 @@ +import urllib2 + +import boto +from boto.exception import S3ResponseError +from boto.s3.key import Key +from boto.s3.connection import OrdinaryCallingFormat + +from freezegun import freeze_time +import requests + +import sure # noqa + +from moto import mock_s3bucket_path + + +def create_connection(key=None, secret=None): + return boto.connect_s3(key, secret, calling_format=OrdinaryCallingFormat()) + + +class MyModel(object): + def __init__(self, name, value): + self.name = name + self.value = value + + def save(self): + conn = create_connection('the_key', 'the_secret') + bucket = conn.get_bucket('mybucket') + k = Key(bucket) + k.key = self.name + k.set_contents_from_string(self.value) + + +@mock_s3bucket_path +def test_my_model_save(): + # Create Bucket so that test can run + conn = create_connection('the_key', 'the_secret') + conn.create_bucket('mybucket') + #################################### + + model_instance = MyModel('steve', 'is awesome') + model_instance.save() + + conn.get_bucket('mybucket').get_key('steve').get_contents_as_string().should.equal('is awesome') + + +@mock_s3bucket_path +def test_missing_key(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + bucket.get_key("the-key").should.equal(None) + + +@mock_s3bucket_path +def test_missing_key_urllib2(): + conn = create_connection('the_key', 'the_secret') + conn.create_bucket("foobar") + + urllib2.urlopen.when.called_with("http://s3.amazonaws.com/foobar/the-key").should.throw(urllib2.HTTPError) + + +@mock_s3bucket_path +def test_empty_key(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("") + + bucket.get_key("the-key").get_contents_as_string().should.equal('') + + +@mock_s3bucket_path +def test_empty_key_set_on_existing_key(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("foobar") + + bucket.get_key("the-key").get_contents_as_string().should.equal('foobar') + + key.set_contents_from_string("") + bucket.get_key("the-key").get_contents_as_string().should.equal('') + + +@mock_s3bucket_path +def test_large_key_save(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("foobar" * 100000) + + bucket.get_key("the-key").get_contents_as_string().should.equal('foobar' * 100000) + + +@mock_s3bucket_path +def test_copy_key(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("some value") + + bucket.copy_key('new-key', 'foobar', 'the-key') + + bucket.get_key("the-key").get_contents_as_string().should.equal("some value") + bucket.get_key("new-key").get_contents_as_string().should.equal("some value") + + +@mock_s3bucket_path +def test_set_metadata(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = 'the-key' + key.set_metadata('md', 'Metadatastring') + key.set_contents_from_string("Testval") + + bucket.get_key('the-key').get_metadata('md').should.equal('Metadatastring') + + +@freeze_time("2012-01-01 12:00:00") +@mock_s3bucket_path +def test_last_modified(): + # See https://github.com/boto/boto/issues/466 + conn = create_connection() + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("some value") + + rs = bucket.get_all_keys() + rs[0].last_modified.should.equal('2012-01-01T12:00:00Z') + + bucket.get_key("the-key").last_modified.should.equal('Sun, 01 Jan 2012 12:00:00 GMT') + + +@mock_s3bucket_path +def test_missing_bucket(): + conn = create_connection('the_key', 'the_secret') + conn.get_bucket.when.called_with('mybucket').should.throw(S3ResponseError) + + +@mock_s3bucket_path +def test_bucket_with_dash(): + conn = create_connection('the_key', 'the_secret') + conn.get_bucket.when.called_with('mybucket-test').should.throw(S3ResponseError) + + +@mock_s3bucket_path +def test_bucket_deletion(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("some value") + + # Try to delete a bucket that still has keys + conn.delete_bucket.when.called_with("foobar").should.throw(S3ResponseError) + + bucket.delete_key("the-key") + conn.delete_bucket("foobar") + + # Get non-existing bucket + conn.get_bucket.when.called_with("foobar").should.throw(S3ResponseError) + + # Delete non-existant bucket + conn.delete_bucket.when.called_with("foobar").should.throw(S3ResponseError) + + +@mock_s3bucket_path +def test_get_all_buckets(): + conn = create_connection('the_key', 'the_secret') + conn.create_bucket("foobar") + conn.create_bucket("foobar2") + buckets = conn.get_all_buckets() + + buckets.should.have.length_of(2) + + +@mock_s3bucket_path +def test_post_to_bucket(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + requests.post("https://s3.amazonaws.com/foobar", { + 'key': 'the-key', + 'file': 'nothing' + }) + + bucket.get_key('the-key').get_contents_as_string().should.equal('nothing') + + +@mock_s3bucket_path +def test_post_with_metadata_to_bucket(): + conn = create_connection('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + requests.post("https://s3.amazonaws.com/foobar", { + 'key': 'the-key', + 'file': 'nothing', + 'x-amz-meta-test': 'metadata' + }) + + bucket.get_key('the-key').get_metadata('test').should.equal('metadata') + + +@mock_s3bucket_path +def test_bucket_method_not_implemented(): + requests.patch.when.called_with("https://s3.amazonaws.com/foobar").should.throw(NotImplementedError) + + +@mock_s3bucket_path +def test_key_method_not_implemented(): + requests.post.when.called_with("https://s3.amazonaws.com/foobar/foo").should.throw(NotImplementedError) + + +@mock_s3bucket_path +def test_bucket_name_with_dot(): + conn = create_connection() + bucket = conn.create_bucket('firstname.lastname') + + k = Key(bucket, 'somekey') + k.set_contents_from_string('somedata') + + +@mock_s3bucket_path +def test_key_with_special_characters(): + conn = create_connection() + bucket = conn.create_bucket('test_bucket_name') + + key = Key(bucket, 'test_list_keys_2/x?y') + key.set_contents_from_string('value1') + + key_list = bucket.list('test_list_keys_2/', '/') + keys = [x for x in key_list] + keys[0].name.should.equal("test_list_keys_2/x?y") + + +@mock_s3bucket_path +def test_bucket_key_listing_order(): + conn = create_connection() + bucket = conn.create_bucket('test_bucket') + prefix = 'toplevel/' + + def store(name): + k = Key(bucket, prefix + name) + k.set_contents_from_string('somedata') + + names = ['x/key', 'y.key1', 'y.key2', 'y.key3', 'x/y/key', 'x/y/z/key'] + + for name in names: + store(name) + + delimiter = None + keys = [x.name for x in bucket.list(prefix, delimiter)] + keys.should.equal([ + 'toplevel/x/key', 'toplevel/x/y/key', 'toplevel/x/y/z/key', + 'toplevel/y.key1', 'toplevel/y.key2', 'toplevel/y.key3' + ]) + + delimiter = '/' + keys = [x.name for x in bucket.list(prefix, delimiter)] + keys.should.equal([ + 'toplevel/y.key1', 'toplevel/y.key2', 'toplevel/y.key3', 'toplevel/x/' + ]) + + # Test delimiter with no prefix + delimiter = '/' + keys = [x.name for x in bucket.list(prefix=None, delimiter=delimiter)] + keys.should.equal(['toplevel']) + + delimiter = None + keys = [x.name for x in bucket.list(prefix + 'x', delimiter)] + keys.should.equal([u'toplevel/x/key', u'toplevel/x/y/key', u'toplevel/x/y/z/key']) + + delimiter = '/' + keys = [x.name for x in bucket.list(prefix + 'x', delimiter)] + keys.should.equal([u'toplevel/x/']) diff --git a/tests/test_s3bucket_path/test_s3bucket_path_utils.py b/tests/test_s3bucket_path/test_s3bucket_path_utils.py new file mode 100644 index 000000000..4b9ff30b1 --- /dev/null +++ b/tests/test_s3bucket_path/test_s3bucket_path_utils.py @@ -0,0 +1,14 @@ +from sure import expect +from moto.s3bucket_path.utils import bucket_name_from_url + + +def test_base_url(): + expect(bucket_name_from_url('https://s3.amazonaws.com/')).should.equal(None) + + +def test_localhost_bucket(): + expect(bucket_name_from_url('https://localhost:5000/wfoobar/abc')).should.equal("wfoobar") + + +def test_localhost_without_bucket(): + expect(bucket_name_from_url('https://www.localhost:5000')).should.equal(None) From df3155c8691f47960d24ea9df6c326c740e793a5 Mon Sep 17 00:00:00 2001 From: Jeff Gregory Date: Fri, 25 Oct 2013 14:36:49 -0700 Subject: [PATCH 17/29] when getting a key (HEAD request) return key.value instead of empty string. This is will mirror boto's method. --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 743039920..57a85fbfe 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -95,6 +95,7 @@ class ResponseObject(object): #Metadata meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) + for form_id in form: result = meta_regex.match(form_id) if result: From d5b3af202ec632795282aceb2a5727b363eb7a45 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 8 Nov 2013 16:05:36 -0500 Subject: [PATCH 18/29] 0.2.10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67e22feb9..05f185a7f 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ if sys.version_info < (2, 7): setup( name='moto', - version='0.2.9', + version='0.2.10', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From bdf5a9e26b129e9dda01b4c550d53c1907ef0b06 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Thu, 14 Nov 2013 17:47:03 +0200 Subject: [PATCH 19/29] convert to bytes for python 2.6 --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 43453e466..7447d0cd9 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -25,7 +25,7 @@ class FakeKey(object): @property def etag(self): value_md5 = hashlib.md5() - value_md5.update(self.value) + value_md5.update(bytes(self.value)) return '"{0}"'.format(value_md5.hexdigest()) @property From a11c80fe201fe9c24f777e545604848773eac4f0 Mon Sep 17 00:00:00 2001 From: jjofseattle Date: Thu, 14 Nov 2013 11:14:14 -0800 Subject: [PATCH 20/29] add route53 --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/route53/__init__.py | 2 + moto/route53/models.py | 57 +++++++++++++ moto/route53/responses.py | 130 +++++++++++++++++++++++++++++ moto/route53/urls.py | 12 +++ requirements.txt | 4 +- tests/test_route53/test_route53.py | 69 +++++++++++++++ 8 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 moto/route53/__init__.py create mode 100644 moto/route53/models.py create mode 100644 moto/route53/responses.py create mode 100644 moto/route53/urls.py create mode 100644 tests/test_route53/test_route53.py diff --git a/moto/__init__.py b/moto/__init__.py index 76cc62c55..634daa00e 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -11,3 +11,4 @@ from .s3bucket_path import mock_s3bucket_path from .ses import mock_ses from .sqs import mock_sqs from .sts import mock_sts +from .route53 import mock_route53 diff --git a/moto/backends.py b/moto/backends.py index 0bc766fe3..b11005227 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -8,6 +8,7 @@ from moto.s3bucket_path import s3bucket_path_backend from moto.ses import ses_backend from moto.sqs import sqs_backend from moto.sts import sts_backend +from moto.route53 import route53_backend BACKENDS = { 'autoscaling': autoscaling_backend, @@ -20,4 +21,5 @@ BACKENDS = { 'ses': ses_backend, 'sqs': sqs_backend, 'sts': sts_backend, + 'route53': route53_backend } diff --git a/moto/route53/__init__.py b/moto/route53/__init__.py new file mode 100644 index 000000000..6448c3c39 --- /dev/null +++ b/moto/route53/__init__.py @@ -0,0 +1,2 @@ +from .models import route53_backend +mock_route53 = route53_backend.decorator diff --git a/moto/route53/models.py b/moto/route53/models.py new file mode 100644 index 000000000..0ae3947ea --- /dev/null +++ b/moto/route53/models.py @@ -0,0 +1,57 @@ +from moto.core import BaseBackend +from moto.core.utils import get_random_hex + + +class FakeZone: + + def __init__(self, name, id): + self.name = name + self.id = id + self.rrsets = {} + + def add_rrset(self, name, rrset): + self.rrsets[name] = rrset + + def delete_rrset(self, name): + del self.rrsets[name] + + +class FakeResourceRecord: + def __init__(self, value): + pass + +class FakeResourceRecordSet: + def __init__(self, name, type, ttl, rrlist): + self.name = name + self.type = type + self.ttl = ttl + self.rrList = rrlist + +class Route53Backend(BaseBackend): + + def __init__(self): + self.zones = {} + + def create_hosted_zone(self, name): + new_id = get_random_hex() + new_zone = FakeZone(name, new_id) + self.zones[new_id] = new_zone + return new_zone + + def get_all_hosted_zones(self): + return self.zones.values() + + def get_hosted_zone(self, id): + return self.zones.get(id) + + def delete_hosted_zone(self, id): + zone = self.zones.get(id) + if zone: + del self.zones[id] + return zone + return None + + +route53_backend = Route53Backend() + + diff --git a/moto/route53/responses.py b/moto/route53/responses.py new file mode 100644 index 000000000..92a4d921c --- /dev/null +++ b/moto/route53/responses.py @@ -0,0 +1,130 @@ +from jinja2 import Template +from urlparse import parse_qs, urlparse +from .models import route53_backend +import xmltodict +import dicttoxml + + +def list_or_create_hostzone_response(request, full_url, headers): + + if request.method == "POST": + r = xmltodict.parse(request.body) + new_zone = route53_backend.create_hosted_zone(r["CreateHostedZoneRequest"]["Name"]) + template = Template(CREATE_HOSTED_ZONE_RESPONSE) + return 201, headers, template.render(zone=new_zone) + + elif request.method == "GET": + all_zones = route53_backend.get_all_hosted_zones() + template = Template(LIST_HOSTED_ZONES_RESPONSE) + return 200, headers, template.render(zones=all_zones) + + +def get_or_delete_hostzone_response(request, full_url, headers): + parsed_url = urlparse(full_url) + zoneid = parsed_url.path.rstrip('/').rsplit('/', 1)[1] + the_zone = route53_backend.get_hosted_zone(zoneid) + if not the_zone: + return 404, headers, "Zone %s not Found" % zoneid + + if request.method == "GET": + template = Template(GET_HOSTED_ZONE_RESPONSE) + return 200, headers, template.render(zone=the_zone) + elif request.method == "DELETE": + route53_backend.delete_hosted_zone(zoneid) + return 200, headers, DELETE_HOSTED_ZONE_RESPONSE + +def rrset_response(request, full_url, headers): + parsed_url = urlparse(full_url) + method = request.method + + zoneid = parsed_url.path.rstrip('/').rsplit('/', 2)[1] + the_zone = route53_backend.get_hosted_zone(zoneid) + if not the_zone: + return 404, headers, "Zone %s Not Found" % zoneid + + if method == "POST": + r = xmltodict.parse(request.body) + for k, v in r['ChangeResourceRecordSetsRequest']['ChangeBatch']['Changes'].items(): + action = v['Action'] + rrset = v['ResourceRecordSet'] + + if action == 'CREATE': + the_zone.add_rrset(rrset["Name"], rrset) + elif action == "DELETE": + the_zone.delete_rrset(rrset["Name"]) + + return 200, headers, CHANGE_RRSET_RESPONSE + + elif method == "GET": + querystring = parse_qs(parsed_url.query) + template = Template(LIST_RRSET_REPONSE) + rrset_list = [] + for key, value in the_zone.rrsets.items(): + if 'type' not in querystring or querystring["type"][0] == value["Type"]: + rrset_list.append(dicttoxml.dicttoxml({"ResourceRecordSet": value}, root=False)) + + return 200, headers, template.render(rrsets=rrset_list) + + + +def not_implemented_response(request, full_url, headers): + parsed_url = urlparse(full_url) + raise NotImplementedError('handling of %s is not yet implemented' % parsed_url.path) + + +LIST_RRSET_REPONSE = """ + + {% for rrset in rrsets %} + {{ rrset }} + {% endfor %} + +""" + +CHANGE_RRSET_RESPONSE = """ + + PENDING + 2010-09-10T01:36:41.958Z + +""" + +DELETE_HOSTED_ZONE_RESPONSE = """ + + +""" + +GET_HOSTED_ZONE_RESPONSE = """ + + /hostedzone/{{ zone.id }} + {{ zone.name }} + {{ zone.rrsets|count }} + + + moto.test.com + +""" + +CREATE_HOSTED_ZONE_RESPONSE = """ + + /hostedzone/{{ zone.id }} + {{ zone.name }} + 0 + + + + moto.test.com + + +""" + +LIST_HOSTED_ZONES_RESPONSE = """ + + {% for zone in zones %} + + {{ zone.id }} + {{ zone.name }} + {{ zone.rrsets|count }} + + {% endfor %} + +""" + diff --git a/moto/route53/urls.py b/moto/route53/urls.py new file mode 100644 index 000000000..6f902e5af --- /dev/null +++ b/moto/route53/urls.py @@ -0,0 +1,12 @@ +import responses + +url_bases = [ + #"https://route53.amazonaws.com/201\d-\d\d-\d\d/hostedzone", + "https://route53.amazonaws.com/201.-..-../hostedzone", +] + +url_paths = { + '{0}$': responses.list_or_create_hostzone_response, + '{0}/.+$': responses.get_or_delete_hostzone_response, + '{0}/.+/rrset$': responses.rrset_response, +} diff --git a/requirements.txt b/requirements.txt index 62f6f0a27..81d8f8e04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,6 @@ mock nose https://github.com/spulec/python-coveralls/tarball/796d9dba34b759664e42ba39e6414209a0f319ad requests -sure \ No newline at end of file +sure +xmltodict +dicttoxml diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py new file mode 100644 index 000000000..465f3b97b --- /dev/null +++ b/tests/test_route53/test_route53.py @@ -0,0 +1,69 @@ +import urllib2 + +import boto +from boto.exception import S3ResponseError +from boto.s3.key import Key +from boto.route53.record import ResourceRecordSets +from freezegun import freeze_time +import requests + +import sure # noqa + +from moto import mock_route53 + + +@mock_route53 +def test_hosted_zone(): + conn = boto.connect_route53('the_key', 'the_secret') + firstzone = conn.create_hosted_zone("testdns.aws.com") + zones = conn.get_all_hosted_zones() + len(zones["ListHostedZonesResponse"]["HostedZones"]).should.equal(1) + + secondzone = conn.create_hosted_zone("testdns1.aws.com") + zones = conn.get_all_hosted_zones() + len(zones["ListHostedZonesResponse"]["HostedZones"]).should.equal(2) + + id1 = firstzone["CreateHostedZoneResponse"]["HostedZone"]["Id"] + zone = conn.get_hosted_zone(id1) + zone["GetHostedZoneResponse"]["HostedZone"]["Name"].should.equal("testdns.aws.com") + + conn.delete_hosted_zone(id1) + zones = conn.get_all_hosted_zones() + len(zones["ListHostedZonesResponse"]["HostedZones"]).should.equal(1) + + conn.get_hosted_zone.when.called_with("abcd").should.throw(boto.route53.exception.DNSServerError, "404 Not Found") + + +@mock_route53 +def test_rrset(): + conn = boto.connect_route53('the_key', 'the_secret') + zone = conn.create_hosted_zone("testdns.aws.com") + zoneid = zone["CreateHostedZoneResponse"]["HostedZone"]["Id"] + + changes = ResourceRecordSets(conn, zoneid) + change = changes.add_change("CREATE", "foo.bar.testdns.aws.com", "A") + change.add_value("1.2.3.4") + changes.commit() + + rrsets = conn.get_all_rrsets(zoneid, type="A") + rrsets.should.have.length_of(1) + rrsets[0].resource_records[0].should.equal('1.2.3.4') + + rrsets = conn.get_all_rrsets(zoneid, type="CNAME") + rrsets.should.have.length_of(0) + + + changes = ResourceRecordSets(conn, zoneid) + changes.add_change("DELETE", "foo.bar.testdns.aws.com", "A") + changes.commit() + + rrsets = conn.get_all_rrsets(zoneid) + rrsets.should.have.length_of(0) + + + + + + + + \ No newline at end of file From 3846c4699018f997b2f18e4af284b118771ca6c4 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Fri, 15 Nov 2013 11:53:39 +0200 Subject: [PATCH 21/29] replace literal ints with constants --- moto/s3/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 7447d0cd9..e59558864 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -7,6 +7,9 @@ from moto.core import BaseBackend from moto.core.utils import iso_8601_datetime, rfc_1123_datetime from .utils import clean_key_name +UPLOAD_ID_BYTES=43 +UPLOAD_PART_MIN_SIZE=5242880 + class FakeKey(object): def __init__(self, name, value): @@ -58,14 +61,14 @@ class FakeMultipart(object): def __init__(self, key_name): self.key_name = key_name self.parts = {} - self.id = base64.b64encode(os.urandom(43)).replace('=', '').replace('+', '') + self.id = base64.b64encode(os.urandom(UPLOAD_ID_BYTES)).replace('=', '').replace('+', '') def complete(self): total = bytearray() last_part_name = len(self.list_parts()) for part in self.list_parts(): - if part.name != last_part_name and len(part.value) < 5242880: + if part.name != last_part_name and len(part.value) < UPLOAD_PART_MIN_SIZE: return total.extend(part.value) From 85e32102fa75e3ef043e43758f4297bbe148a47c Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Fri, 15 Nov 2013 11:59:30 +0200 Subject: [PATCH 22/29] break multipart test in two --- tests/test_s3/test_s3.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 8cbd489d0..3d9e3f1fb 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -39,7 +39,7 @@ def test_my_model_save(): @mock_s3 -def test_multipart_upload(): +def test_multipart_upload_too_small(): conn = boto.connect_s3('the_key', 'the_secret') bucket = conn.create_bucket("foobar") @@ -49,12 +49,20 @@ def test_multipart_upload(): # Multipart with total size under 5MB is refused multipart.complete_upload.should.throw(S3ResponseError) + +@mock_s3 +def test_multipart_upload(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + multipart = bucket.initiate_multipart_upload("the-key") part1 = '0' * 5242880 - multipart.upload_part_from_file(BytesIO('0' * 5242880), 1) + multipart.upload_part_from_file(BytesIO(part1), 1) + # last part, can be less than 5 MB part2 = '1' - multipart.upload_part_from_file(BytesIO('1'), 2) + multipart.upload_part_from_file(BytesIO(part2), 2) multipart.complete_upload() + # we should get both parts as the key contents bucket.get_key("the-key").get_contents_as_string().should.equal(part1 + part2) From 8da1d31432f5f0e1a54742ecb0e538bc92b03dfe Mon Sep 17 00:00:00 2001 From: jjofseattle Date: Fri, 15 Nov 2013 15:29:01 -0800 Subject: [PATCH 23/29] fix style issues --- moto/route53/models.py | 4 ---- moto/route53/responses.py | 18 ++++++------------ moto/route53/urls.py | 1 - tests/test_route53/test_route53.py | 8 -------- 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 0ae3947ea..39ee4fc6d 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -16,10 +16,6 @@ class FakeZone: del self.rrsets[name] -class FakeResourceRecord: - def __init__(self, value): - pass - class FakeResourceRecordSet: def __init__(self, name, type, ttl, rrlist): self.name = name diff --git a/moto/route53/responses.py b/moto/route53/responses.py index 92a4d921c..c2527320e 100644 --- a/moto/route53/responses.py +++ b/moto/route53/responses.py @@ -8,8 +8,8 @@ import dicttoxml def list_or_create_hostzone_response(request, full_url, headers): if request.method == "POST": - r = xmltodict.parse(request.body) - new_zone = route53_backend.create_hosted_zone(r["CreateHostedZoneRequest"]["Name"]) + elements = xmltodict.parse(request.body) + new_zone = route53_backend.create_hosted_zone(elements["CreateHostedZoneRequest"]["Name"]) template = Template(CREATE_HOSTED_ZONE_RESPONSE) return 201, headers, template.render(zone=new_zone) @@ -43,10 +43,10 @@ def rrset_response(request, full_url, headers): return 404, headers, "Zone %s Not Found" % zoneid if method == "POST": - r = xmltodict.parse(request.body) - for k, v in r['ChangeResourceRecordSetsRequest']['ChangeBatch']['Changes'].items(): - action = v['Action'] - rrset = v['ResourceRecordSet'] + elements = xmltodict.parse(request.body) + for key, value in elements['ChangeResourceRecordSetsRequest']['ChangeBatch']['Changes'].items(): + action = value['Action'] + rrset = value['ResourceRecordSet'] if action == 'CREATE': the_zone.add_rrset(rrset["Name"], rrset) @@ -64,12 +64,6 @@ def rrset_response(request, full_url, headers): rrset_list.append(dicttoxml.dicttoxml({"ResourceRecordSet": value}, root=False)) return 200, headers, template.render(rrsets=rrset_list) - - - -def not_implemented_response(request, full_url, headers): - parsed_url = urlparse(full_url) - raise NotImplementedError('handling of %s is not yet implemented' % parsed_url.path) LIST_RRSET_REPONSE = """ diff --git a/moto/route53/urls.py b/moto/route53/urls.py index 6f902e5af..7b76e6b23 100644 --- a/moto/route53/urls.py +++ b/moto/route53/urls.py @@ -1,7 +1,6 @@ import responses url_bases = [ - #"https://route53.amazonaws.com/201\d-\d\d-\d\d/hostedzone", "https://route53.amazonaws.com/201.-..-../hostedzone", ] diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 465f3b97b..ceffa63e0 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -59,11 +59,3 @@ def test_rrset(): rrsets = conn.get_all_rrsets(zoneid) rrsets.should.have.length_of(0) - - - - - - - - \ No newline at end of file From 08777e4b182378d392bb5307afe85745e01705e4 Mon Sep 17 00:00:00 2001 From: jjofseattle Date: Fri, 15 Nov 2013 15:35:46 -0800 Subject: [PATCH 24/29] pep8 --- moto/route53/models.py | 4 ++-- moto/route53/responses.py | 4 ++-- tests/test_route53/test_route53.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 39ee4fc6d..4afcfe922 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -17,12 +17,14 @@ class FakeZone: class FakeResourceRecordSet: + def __init__(self, name, type, ttl, rrlist): self.name = name self.type = type self.ttl = ttl self.rrList = rrlist + class Route53Backend(BaseBackend): def __init__(self): @@ -49,5 +51,3 @@ class Route53Backend(BaseBackend): route53_backend = Route53Backend() - - diff --git a/moto/route53/responses.py b/moto/route53/responses.py index c2527320e..55160922e 100644 --- a/moto/route53/responses.py +++ b/moto/route53/responses.py @@ -33,6 +33,7 @@ def get_or_delete_hostzone_response(request, full_url, headers): route53_backend.delete_hosted_zone(zoneid) return 200, headers, DELETE_HOSTED_ZONE_RESPONSE + def rrset_response(request, full_url, headers): parsed_url = urlparse(full_url) method = request.method @@ -120,5 +121,4 @@ LIST_HOSTED_ZONES_RESPONSE = """""" diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index ceffa63e0..3e6132833 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -23,7 +23,7 @@ def test_hosted_zone(): zones = conn.get_all_hosted_zones() len(zones["ListHostedZonesResponse"]["HostedZones"]).should.equal(2) - id1 = firstzone["CreateHostedZoneResponse"]["HostedZone"]["Id"] + id1 = firstzone["CreateHostedZoneResponse"]["HostedZone"]["Id"] zone = conn.get_hosted_zone(id1) zone["GetHostedZoneResponse"]["HostedZone"]["Name"].should.equal("testdns.aws.com") @@ -52,7 +52,6 @@ def test_rrset(): rrsets = conn.get_all_rrsets(zoneid, type="CNAME") rrsets.should.have.length_of(0) - changes = ResourceRecordSets(conn, zoneid) changes.add_change("DELETE", "foo.bar.testdns.aws.com", "A") changes.commit() From 2d6e64924589a4a6eed7ec0c8bdb543a90a7e75b Mon Sep 17 00:00:00 2001 From: jjofseattle Date: Fri, 15 Nov 2013 16:20:25 -0800 Subject: [PATCH 25/29] improve coverage --- moto/route53/models.py | 9 --------- tests/test_route53/test_route53.py | 4 ++++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 4afcfe922..0ee7dc446 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -16,15 +16,6 @@ class FakeZone: del self.rrsets[name] -class FakeResourceRecordSet: - - def __init__(self, name, type, ttl, rrlist): - self.name = name - self.type = type - self.ttl = ttl - self.rrList = rrlist - - class Route53Backend(BaseBackend): def __init__(self): diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index 3e6132833..57da8112a 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -37,6 +37,10 @@ def test_hosted_zone(): @mock_route53 def test_rrset(): conn = boto.connect_route53('the_key', 'the_secret') + + conn.get_all_rrsets.when.called_with("abcd", type="A").\ + should.throw(boto.route53.exception.DNSServerError, "404 Not Found") + zone = conn.create_hosted_zone("testdns.aws.com") zoneid = zone["CreateHostedZoneResponse"]["HostedZone"]["Id"] From 38b26f038fb7855a6b4d80fad897997acd4f0ef2 Mon Sep 17 00:00:00 2001 From: jjofseattle Date: Wed, 20 Nov 2013 14:45:44 -0800 Subject: [PATCH 26/29] handle double deletion caused by httpretty --- moto/route53/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 0ee7dc446..d901996fa 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -13,7 +13,7 @@ class FakeZone: self.rrsets[name] = rrset def delete_rrset(self, name): - del self.rrsets[name] + self.rrsets.pop(name, None) class Route53Backend(BaseBackend): From 0b7ace7a5f154d1132f28cbe1a03365bffa0a0fd Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 23 Nov 2013 10:29:13 -0500 Subject: [PATCH 27/29] Add more contributors from multipart s3 pull request --- AUTHORS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AUTHORS.md b/AUTHORS.md index 1cf19dd76..de484e1e2 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -7,4 +7,7 @@ Moto is written by Steve Pulec with contributions from: * [Dan Berglund](https://github.com/cheif) * [Lincoln de Sousa](https://github.com/clarete) * [mhock](https://github.com/mhock) -* [Ilya Sukhanov](https://github.com/IlyaSukhanov) \ No newline at end of file +* [Ilya Sukhanov](https://github.com/IlyaSukhanov) +* [Lucian Branescu Mihaila](https://github.com/lucian1900) +* [Konstantinos Koukopoulos](https://github.com/kouk) +* [attili](https://github.com/attili) From ecb7fa3a667f66b98eee0c2c4186da1cc0df46e5 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 23 Nov 2013 10:38:04 -0500 Subject: [PATCH 28/29] Add to authors --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index de484e1e2..8a0831dcd 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -11,3 +11,4 @@ Moto is written by Steve Pulec with contributions from: * [Lucian Branescu Mihaila](https://github.com/lucian1900) * [Konstantinos Koukopoulos](https://github.com/kouk) * [attili](https://github.com/attili) +* [JJ Zeng](https://github.com/jjofseattle) From ac1d2f5ef4e2ec968e3372c5ef40a6073f1249ba Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 23 Nov 2013 10:39:32 -0500 Subject: [PATCH 29/29] 0.2.11 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 05f185a7f..4a5806ca9 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ if sys.version_info < (2, 7): setup( name='moto', - version='0.2.10', + version='0.2.11', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec',