From 1df454a6329f9b368d46d931616f738ebfbce33c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 27 Nov 2015 13:49:44 -0500 Subject: [PATCH 1/2] first working version of s3 refactor. --- moto/s3/responses.py | 57 +++++++++++++++---- moto/s3/urls.py | 15 ++++- moto/s3bucket_path/__init__.py | 7 ++- moto/s3bucket_path/responses.py | 10 +--- moto/s3bucket_path/utils.py | 8 +++ .../test_bucket_path_server.py | 22 +++---- 6 files changed, 85 insertions(+), 34 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 27a6d4536..4ee6e9659 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -8,6 +8,9 @@ import xmltodict from moto.core.responses import _TemplateEnvironmentMixin +from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys + + from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl from .utils import bucket_name_from_url, metadata_from_headers @@ -21,19 +24,25 @@ def parse_key_name(pth): return pth.lstrip("/") +def is_delete_keys(path, bucket_name): + return path == u'/?delete' + + class ResponseObject(_TemplateEnvironmentMixin): - def __init__(self, backend, bucket_name_from_url, parse_key_name, + def __init__(self, backend, parse_key_name, bucket_name_from_url, is_delete_keys=None): super(ResponseObject, self).__init__() self.backend = backend - self.bucket_name_from_url = bucket_name_from_url - self.parse_key_name = parse_key_name - if is_delete_keys: - self.is_delete_keys = is_delete_keys + # self.bucket_name_from_url = bucket_name_from_url + # self.parse_key_name = parse_key_name + # if is_delete_keys: + # self.is_delete_keys = is_delete_keys - @staticmethod - def is_delete_keys(path, bucket_name): - return path == u'/?delete' + def is_delete_keys(self, request, path, bucket_name): + if self.is_path_based_buckets(request): + return bucketpath_is_delete_keys(path, bucket_name) + else: + return is_delete_keys(path, bucket_name) def all_buckets(self): # No bucket specified. Listing all buckets @@ -41,6 +50,30 @@ class ResponseObject(_TemplateEnvironmentMixin): template = self.response_template(S3_ALL_BUCKETS) return template.render(buckets=all_buckets) + def is_path_based_buckets(self, request): + return request.headers['host'] == 's3.amazonaws.com' + + def parse_bucket_name_from_url(self, request, url): + if self.is_path_based_buckets(request): + return bucketpath_bucket_name_from_url(url) + else: + return bucket_name_from_url(url) + + def parse_key_name(self, request, url): + if self.is_path_based_buckets(request): + return bucketpath_parse_key_name(url) + else: + return parse_key_name(url) + + def response(self, request, full_url, headers): + # Depending on which calling format the client is using, we don't know + # if this is a bucket or key request so we have to check + if self.is_path_based_buckets(request): + # Using path-based buckets + return self.bucket_response(request, full_url, headers) + else: + return self.key_response(request, full_url, headers) + def bucket_response(self, request, full_url, headers): try: response = self._bucket_response(request, full_url, headers) @@ -62,7 +95,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if region_match: region_name = region_match.groups()[0] - bucket_name = self.bucket_name_from_url(full_url) + bucket_name = self.parse_bucket_name_from_url(request, full_url) if not bucket_name: # If no bucket specified, list all buckets return self.all_buckets() @@ -232,7 +265,7 @@ class ResponseObject(_TemplateEnvironmentMixin): return 409, headers, template.render(bucket=removed_bucket) def _bucket_response_post(self, request, bucket_name, headers): - if self.is_delete_keys(request.path, bucket_name): + if self.is_delete_keys(request, request.path, bucket_name): return self._bucket_response_delete_keys(request, bucket_name, headers) # POST to bucket-url should create file from form @@ -320,8 +353,8 @@ class ResponseObject(_TemplateEnvironmentMixin): query = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method - key_name = self.parse_key_name(parsed_url.path) - bucket_name = self.bucket_name_from_url(full_url) + key_name = self.parse_key_name(request, parsed_url.path) + bucket_name = self.parse_bucket_name_from_url(request, full_url) if hasattr(request, 'body'): # Boto diff --git a/moto/s3/urls.py b/moto/s3/urls.py index c775d02c7..f12ba4e9b 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -2,10 +2,23 @@ from __future__ import unicode_literals from .responses import S3ResponseInstance url_bases = [ + "https?://s3(.*).amazonaws.com", "https?://(?P[a-zA-Z0-9\-_.]*)\.?s3(.*).amazonaws.com" ] url_paths = { + # subdomain bucket '{0}/$': S3ResponseInstance.bucket_response, - '{0}/(?P.+)': S3ResponseInstance.key_response, + + # subdomain key of path-based bucket + '{0}/(?P.+)': S3ResponseInstance.response, + + + # path-based bucket + key + '{0}/(?P[a-zA-Z0-9\-_./]+)/(?P.+)': S3ResponseInstance.key_response, + + + + # '{0}/(?P[a-zA-Z0-9\-_.]+)$': ro.bucket_response, + # '{0}/(?P[a-zA-Z0-9\-_.]+)/$': bucket_response2, } diff --git a/moto/s3bucket_path/__init__.py b/moto/s3bucket_path/__init__.py index fdeeaecb8..869329745 100644 --- a/moto/s3bucket_path/__init__.py +++ b/moto/s3bucket_path/__init__.py @@ -1,3 +1,6 @@ from __future__ import unicode_literals -from .models import s3bucket_path_backend -mock_s3bucket_path = s3bucket_path_backend.decorator +# from .models import s3bucket_path_backend +from moto import mock_s3 + +# mock_s3bucket_path = s3bucket_path_backend.decorator +mock_s3bucket_path = mock_s3 diff --git a/moto/s3bucket_path/responses.py b/moto/s3bucket_path/responses.py index 6de82fbb0..359d71704 100644 --- a/moto/s3bucket_path/responses.py +++ b/moto/s3bucket_path/responses.py @@ -1,19 +1,11 @@ from __future__ import unicode_literals from .models import s3bucket_path_backend -from .utils import bucket_name_from_url +from .utils import bucket_name_from_url, parse_key_name, is_delete_keys from moto.s3.responses import ResponseObject -def parse_key_name(pth): - return "/".join(pth.rstrip("/").split("/")[2:]) - - -def is_delete_keys(path, bucket_name): - return path == u'/' + bucket_name + u'/?delete' - - S3BucketPathResponseInstance = ResponseObject( s3bucket_path_backend, bucket_name_from_url, diff --git a/moto/s3bucket_path/utils.py b/moto/s3bucket_path/utils.py index c76a98e12..933c4f6d4 100644 --- a/moto/s3bucket_path/utils.py +++ b/moto/s3bucket_path/utils.py @@ -9,3 +9,11 @@ def bucket_name_from_url(url): if len(l) == 0 or l[0] == "": return None return l[0] + + +def parse_key_name(path): + return "/".join(path.rstrip("/").split("/")[2:]) + + +def is_delete_keys(path, bucket_name): + return path == u'/' + bucket_name + u'/?delete' diff --git a/tests/test_s3bucket_path/test_bucket_path_server.py b/tests/test_s3bucket_path/test_bucket_path_server.py index d639c4ce6..306d12285 100644 --- a/tests/test_s3bucket_path/test_bucket_path_server.py +++ b/tests/test_s3bucket_path/test_bucket_path_server.py @@ -7,12 +7,14 @@ import moto.server as server Test the different server responses ''' +HEADERS = {'host': 's3.amazonaws.com'} + def test_s3_server_get(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.get('/') + res = test_client.get('/', headers=HEADERS) res.data.should.contain(b'ListAllMyBucketsResult') @@ -21,23 +23,23 @@ def test_s3_server_bucket_create(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.put('/foobar/', 'http://localhost:5000') + res = test_client.put('/foobar/', 'http://localhost:5000', headers=HEADERS) res.status_code.should.equal(200) - res = test_client.get('/') + res = test_client.get('/', headers=HEADERS) res.data.should.contain(b'foobar') - res = test_client.get('/foobar/', 'http://localhost:5000') + res = test_client.get('/foobar/', 'http://localhost:5000', headers=HEADERS) res.status_code.should.equal(200) res.data.should.contain(b"ListBucketResult") - res = test_client.get('/missing-bucket/', 'http://localhost:5000') + res = test_client.get('/missing-bucket/', 'http://localhost:5000', headers=HEADERS) res.status_code.should.equal(404) - res = test_client.put('/foobar/bar/', 'http://localhost:5000', data='test value') + res = test_client.put('/foobar/bar/', 'http://localhost:5000', data='test value', headers=HEADERS) res.status_code.should.equal(200) - res = test_client.get('/foobar/bar/', 'http://localhost:5000') + res = test_client.get('/foobar/bar/', 'http://localhost:5000', headers=HEADERS) res.status_code.should.equal(200) res.data.should.equal(b"test value") @@ -46,14 +48,14 @@ def test_s3_server_post_to_bucket(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.put('/foobar2/', 'http://localhost:5000/') + res = test_client.put('/foobar2/', 'http://localhost:5000/', headers=HEADERS) res.status_code.should.equal(200) test_client.post('/foobar2/', "https://localhost:5000/", data={ 'key': 'the-key', 'file': 'nothing' - }) + }, headers=HEADERS) - res = test_client.get('/foobar2/the-key/', 'http://localhost:5000/') + res = test_client.get('/foobar2/the-key/', 'http://localhost:5000/', headers=HEADERS) res.status_code.should.equal(200) res.data.should.equal(b"nothing") From 0df03ba409b4b3baf4e3492d84ac2d4d1d15a7a1 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 27 Nov 2015 14:43:03 -0500 Subject: [PATCH 2/2] cleanup code. --- moto/backends.py | 3 +- moto/s3/responses.py | 49 +++++++++---------- moto/s3/urls.py | 8 +-- moto/s3bucket_path/__init__.py | 4 +- moto/s3bucket_path/models.py | 8 --- moto/s3bucket_path/responses.py | 14 ------ moto/s3bucket_path/urls.py | 21 -------- tests/test_s3/test_s3.py | 10 ++++ .../test_bucket_path_server.py | 22 ++++----- 9 files changed, 47 insertions(+), 92 deletions(-) delete mode 100644 moto/s3bucket_path/models.py delete mode 100644 moto/s3bucket_path/responses.py delete mode 100644 moto/s3bucket_path/urls.py diff --git a/moto/backends.py b/moto/backends.py index cb040ab93..83dfbc00d 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -15,7 +15,6 @@ from moto.kms import kms_backend from moto.rds import rds_backend from moto.redshift import redshift_backend from moto.s3 import s3_backend -from moto.s3bucket_path import s3bucket_path_backend from moto.ses import ses_backend from moto.sns import sns_backend from moto.sqs import sqs_backend @@ -39,7 +38,7 @@ BACKENDS = { 'redshift': redshift_backend, 'rds': rds_backend, 's3': s3_backend, - 's3bucket_path': s3bucket_path_backend, + 's3bucket_path': s3_backend, 'ses': ses_backend, 'sns': sns_backend, 'sqs': sqs_backend, diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 4ee6e9659..18698c275 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -29,20 +29,9 @@ def is_delete_keys(path, bucket_name): class ResponseObject(_TemplateEnvironmentMixin): - def __init__(self, backend, parse_key_name, bucket_name_from_url, - is_delete_keys=None): + def __init__(self, backend): super(ResponseObject, self).__init__() self.backend = backend - # self.bucket_name_from_url = bucket_name_from_url - # self.parse_key_name = parse_key_name - # if is_delete_keys: - # self.is_delete_keys = is_delete_keys - - def is_delete_keys(self, request, path, bucket_name): - if self.is_path_based_buckets(request): - return bucketpath_is_delete_keys(path, bucket_name) - else: - return is_delete_keys(path, bucket_name) def all_buckets(self): # No bucket specified. Listing all buckets @@ -50,29 +39,39 @@ class ResponseObject(_TemplateEnvironmentMixin): template = self.response_template(S3_ALL_BUCKETS) return template.render(buckets=all_buckets) - def is_path_based_buckets(self, request): - return request.headers['host'] == 's3.amazonaws.com' + def subdomain_based_buckets(self, request): + host = request.headers['host'] + if host.startswith("localhost"): + # For localhost, default to path-based buckets + return False + return host != 's3.amazonaws.com' and not re.match("s3.(.*).amazonaws.com", host) + + def is_delete_keys(self, request, path, bucket_name): + if self.subdomain_based_buckets(request): + return is_delete_keys(path, bucket_name) + else: + return bucketpath_is_delete_keys(path, bucket_name) def parse_bucket_name_from_url(self, request, url): - if self.is_path_based_buckets(request): - return bucketpath_bucket_name_from_url(url) - else: + if self.subdomain_based_buckets(request): return bucket_name_from_url(url) + else: + return bucketpath_bucket_name_from_url(url) def parse_key_name(self, request, url): - if self.is_path_based_buckets(request): - return bucketpath_parse_key_name(url) - else: + if self.subdomain_based_buckets(request): return parse_key_name(url) + else: + return bucketpath_parse_key_name(url) - def response(self, request, full_url, headers): + def ambiguous_response(self, request, full_url, headers): # Depending on which calling format the client is using, we don't know # if this is a bucket or key request so we have to check - if self.is_path_based_buckets(request): + if self.subdomain_based_buckets(request): + return self.key_response(request, full_url, headers) + else: # Using path-based buckets return self.bucket_response(request, full_url, headers) - else: - return self.key_response(request, full_url, headers) def bucket_response(self, request, full_url, headers): try: @@ -559,7 +558,7 @@ class ResponseObject(_TemplateEnvironmentMixin): else: raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far") -S3ResponseInstance = ResponseObject(s3_backend, bucket_name_from_url, parse_key_name) +S3ResponseInstance = ResponseObject(s3_backend) S3_ALL_BUCKETS = """ diff --git a/moto/s3/urls.py b/moto/s3/urls.py index f12ba4e9b..dda4b273a 100644 --- a/moto/s3/urls.py +++ b/moto/s3/urls.py @@ -11,14 +11,8 @@ url_paths = { '{0}/$': S3ResponseInstance.bucket_response, # subdomain key of path-based bucket - '{0}/(?P.+)': S3ResponseInstance.response, - + '{0}/(?P.+)': S3ResponseInstance.ambiguous_response, # path-based bucket + key '{0}/(?P[a-zA-Z0-9\-_./]+)/(?P.+)': S3ResponseInstance.key_response, - - - - # '{0}/(?P[a-zA-Z0-9\-_.]+)$': ro.bucket_response, - # '{0}/(?P[a-zA-Z0-9\-_.]+)/$': bucket_response2, } diff --git a/moto/s3bucket_path/__init__.py b/moto/s3bucket_path/__init__.py index 869329745..85031a06e 100644 --- a/moto/s3bucket_path/__init__.py +++ b/moto/s3bucket_path/__init__.py @@ -1,6 +1,4 @@ from __future__ import unicode_literals -# from .models import s3bucket_path_backend -from moto import mock_s3 -# mock_s3bucket_path = s3bucket_path_backend.decorator +from moto import mock_s3 mock_s3bucket_path = mock_s3 diff --git a/moto/s3bucket_path/models.py b/moto/s3bucket_path/models.py deleted file mode 100644 index ec991d9d4..000000000 --- a/moto/s3bucket_path/models.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import unicode_literals -from moto.s3.models import S3Backend - - -class S3BucketPathBackend(S3Backend): - pass - -s3bucket_path_backend = S3BucketPathBackend() diff --git a/moto/s3bucket_path/responses.py b/moto/s3bucket_path/responses.py deleted file mode 100644 index 359d71704..000000000 --- a/moto/s3bucket_path/responses.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import unicode_literals -from .models import s3bucket_path_backend - -from .utils import bucket_name_from_url, parse_key_name, is_delete_keys - -from moto.s3.responses import ResponseObject - - -S3BucketPathResponseInstance = ResponseObject( - s3bucket_path_backend, - bucket_name_from_url, - parse_key_name, - is_delete_keys, -) diff --git a/moto/s3bucket_path/urls.py b/moto/s3bucket_path/urls.py deleted file mode 100644 index c5dc86f2f..000000000 --- a/moto/s3bucket_path/urls.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import unicode_literals -from .responses import S3BucketPathResponseInstance as ro - -url_bases = [ - "https?://s3(.*).amazonaws.com" -] - - -def bucket_response2(*args): - return ro.bucket_response(*args) - - -def bucket_response3(*args): - return ro.bucket_response(*args) - -url_paths = { - '{0}/$': bucket_response3, - '{0}/(?P[a-zA-Z0-9\-_.]+)$': ro.bucket_response, - '{0}/(?P[a-zA-Z0-9\-_.]+)/$': bucket_response2, - '{0}/(?P[a-zA-Z0-9\-_./]+)/(?P.+)': ro.key_response -} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index aff033c5f..e2de0c473 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -961,6 +961,16 @@ def test_boto3_bucket_create(): s3.Object('blah', 'hello.txt').get()['Body'].read().decode("utf-8").should.equal("some text") +@mock_s3 +def test_boto3_bucket_create_eu_central(): + s3 = boto3.resource('s3', region_name='eu-central-1') + s3.create_bucket(Bucket="blah") + + s3.Object('blah', 'hello.txt').put(Body="some text") + + s3.Object('blah', 'hello.txt').get()['Body'].read().decode("utf-8").should.equal("some text") + + @mock_s3 def test_boto3_head_object(): s3 = boto3.resource('s3', region_name='us-east-1') diff --git a/tests/test_s3bucket_path/test_bucket_path_server.py b/tests/test_s3bucket_path/test_bucket_path_server.py index 306d12285..d639c4ce6 100644 --- a/tests/test_s3bucket_path/test_bucket_path_server.py +++ b/tests/test_s3bucket_path/test_bucket_path_server.py @@ -7,14 +7,12 @@ import moto.server as server Test the different server responses ''' -HEADERS = {'host': 's3.amazonaws.com'} - def test_s3_server_get(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.get('/', headers=HEADERS) + res = test_client.get('/') res.data.should.contain(b'ListAllMyBucketsResult') @@ -23,23 +21,23 @@ def test_s3_server_bucket_create(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.put('/foobar/', 'http://localhost:5000', headers=HEADERS) + res = test_client.put('/foobar/', 'http://localhost:5000') res.status_code.should.equal(200) - res = test_client.get('/', headers=HEADERS) + res = test_client.get('/') res.data.should.contain(b'foobar') - res = test_client.get('/foobar/', 'http://localhost:5000', headers=HEADERS) + res = test_client.get('/foobar/', 'http://localhost:5000') res.status_code.should.equal(200) res.data.should.contain(b"ListBucketResult") - res = test_client.get('/missing-bucket/', 'http://localhost:5000', headers=HEADERS) + res = test_client.get('/missing-bucket/', 'http://localhost:5000') res.status_code.should.equal(404) - res = test_client.put('/foobar/bar/', 'http://localhost:5000', data='test value', headers=HEADERS) + res = test_client.put('/foobar/bar/', 'http://localhost:5000', data='test value') res.status_code.should.equal(200) - res = test_client.get('/foobar/bar/', 'http://localhost:5000', headers=HEADERS) + res = test_client.get('/foobar/bar/', 'http://localhost:5000') res.status_code.should.equal(200) res.data.should.equal(b"test value") @@ -48,14 +46,14 @@ def test_s3_server_post_to_bucket(): backend = server.create_backend_app("s3bucket_path") test_client = backend.test_client() - res = test_client.put('/foobar2/', 'http://localhost:5000/', headers=HEADERS) + res = test_client.put('/foobar2/', 'http://localhost:5000/') res.status_code.should.equal(200) test_client.post('/foobar2/', "https://localhost:5000/", data={ 'key': 'the-key', 'file': 'nothing' - }, headers=HEADERS) + }) - res = test_client.get('/foobar2/the-key/', 'http://localhost:5000/', headers=HEADERS) + res = test_client.get('/foobar2/the-key/', 'http://localhost:5000/') res.status_code.should.equal(200) res.data.should.equal(b"nothing")