From 3880be5ea9944dcfaf62794158e356e4ee3651bb Mon Sep 17 00:00:00 2001 From: Dan Berglund Date: Tue, 14 May 2013 19:47:24 +0200 Subject: [PATCH 1/3] Added support for metadata on files, and support for POST:ing files to S3 --- moto/s3/models.py | 11 +++++++++++ moto/s3/responses.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 69ef827ff..15d5fbe67 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -10,6 +10,13 @@ class FakeKey(object): self.name = name self.value = value self.last_modified = datetime.datetime.now() + self._metadata = {} + + def set_metadata(self, key, metadata): + self._metadata[key] = metadata + + def get_metadata(self, key): + return self._metadata[key] def append_to_value(self, value): self.value += value @@ -31,6 +38,10 @@ class FakeKey(object): # https://github.com/boto/boto/issues/466 RFC1123 = '%a, %d %b %Y %H:%M:%S GMT' return self.last_modified.strftime(RFC1123) + + @property + def metadata(self): + return self._metadata @property def response_dict(self): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 766c06e9f..092379b82 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1,4 +1,5 @@ from urlparse import parse_qs, urlparse +import re from jinja2 import Template @@ -67,6 +68,22 @@ def _bucket_response(request, full_url, headers): # Tried to delete a bucket that still has keys template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR) return 409, headers, template.render(bucket=removed_bucket) + elif method == 'POST': + #POST to bucket-url should create file from form + key = request.form['key'] + f = request.form['file'] + new_key = s3_backend.set_key(bucket_name, key, "") + #TODO Set actual file + + #Metadata + meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) + for form_id in request.form: + result = meta_regex.match(form_id) + if result: + meta_key = result.group(0).lower() + metadata = request.form[form_id] + new_key.set_metadata(meta_key, metadata) + return 200, headers, "" else: raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method)) @@ -84,8 +101,8 @@ def _key_response(request, full_url, headers): parsed_url = urlparse(full_url) method = request.method - key_name = parsed_url.path.lstrip('/') bucket_name = bucket_name_from_url(full_url) + key_name = parsed_url.path.split(bucket_name + '/')[-1] if hasattr(request, 'body'): # Boto body = request.body @@ -96,7 +113,8 @@ def _key_response(request, full_url, headers): if method == 'GET': key = s3_backend.get_key(bucket_name, key_name) if key: - return key.value + headers.update(key.metadata) + return 200, headers, key.value else: return 404, headers, "" if method == 'PUT': @@ -118,6 +136,15 @@ def _key_response(request, full_url, headers): # Initial data new_key = s3_backend.set_key(bucket_name, key_name, body) request.streaming = True + + #Metadata + meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) + for header in request.headers: + result = meta_regex.match(header[0]) + if result: + meta_key = result.group(0).lower() + metadata = header[1] + new_key.set_metadata(meta_key, metadata) template = Template(S3_OBJECT_RESPONSE) headers.update(new_key.response_dict) return 200, headers, template.render(key=new_key) @@ -125,7 +152,7 @@ def _key_response(request, full_url, headers): key = s3_backend.get_key(bucket_name, key_name) if key: headers.update(key.response_dict) - return 200, headers, S3_OBJECT_RESPONSE + return 200, headers, "" else: return 404, headers, "" elif method == 'DELETE': From d8e9301c54b9e68c9ee7cd9a16e484e4ab111162 Mon Sep 17 00:00:00 2001 From: Dan Berglund Date: Wed, 15 May 2013 08:55:25 +0200 Subject: [PATCH 2/3] Added metadata to HEAD-response, boto uses this when only metadata is fetched --- moto/s3/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 092379b82..23cbb5410 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -151,6 +151,7 @@ def _key_response(request, full_url, headers): elif method == 'HEAD': key = s3_backend.get_key(bucket_name, key_name) if key: + headers.update(key.metadata) headers.update(key.response_dict) return 200, headers, "" else: From 7de4399b93a3b258c9567a4b7bd18fd1d665cfae Mon Sep 17 00:00:00 2001 From: Dan Berglund Date: Fri, 17 May 2013 11:43:09 +0200 Subject: [PATCH 3/3] Added tests and made current tests pass --- moto/s3/responses.py | 35 +++++++++++++++++++++++------------ tests/test_s3/test_s3.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 23cbb5410..2062ccee9 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -70,18 +70,28 @@ def _bucket_response(request, full_url, headers): return 409, headers, template.render(bucket=removed_bucket) elif method == 'POST': #POST to bucket-url should create file from form - key = request.form['key'] - f = request.form['file'] - new_key = s3_backend.set_key(bucket_name, key, "") - #TODO Set actual file + if hasattr(request, 'form'): + #Not HTTPretty + form = request.form + else: + #HTTPretty, build new form object + form = {} + for kv in request.body.split('&'): + k, v = kv.split('=') + form[k] = v + + key = form['key'] + f = form['file'] + + new_key = s3_backend.set_key(bucket_name, key, f) #Metadata meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) - for form_id in request.form: + for form_id in form: result = meta_regex.match(form_id) if result: meta_key = result.group(0).lower() - metadata = request.form[form_id] + metadata = form[form_id] new_key.set_metadata(meta_key, metadata) return 200, headers, "" else: @@ -101,8 +111,8 @@ def _key_response(request, full_url, headers): parsed_url = urlparse(full_url) method = request.method + key_name = parsed_url.path.lstrip('/') bucket_name = bucket_name_from_url(full_url) - key_name = parsed_url.path.split(bucket_name + '/')[-1] if hasattr(request, 'body'): # Boto body = request.body @@ -140,11 +150,12 @@ def _key_response(request, full_url, headers): #Metadata meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE) for header in request.headers: - result = meta_regex.match(header[0]) - if result: - meta_key = result.group(0).lower() - metadata = header[1] - new_key.set_metadata(meta_key, metadata) + if isinstance(header, basestring): + result = meta_regex.match(header) + if result: + meta_key = result.group(0).lower() + metadata = request.headers[header] + new_key.set_metadata(meta_key, metadata) template = Template(S3_OBJECT_RESPONSE) headers.update(new_key.response_dict) return 200, headers, template.render(key=new_key) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 54da1b9ac..1107c4189 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -101,7 +101,17 @@ def test_copy_key(): bucket.get_key("the-key").get_contents_as_string().should.equal("some value") bucket.get_key("new-key").get_contents_as_string().should.equal("some value") + +@mock_s3 +def test_set_metadata(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = 'the-key' + key.set_metadata('md', 'Metadatastring') + key.set_contents_from_string("Testval") + bucket.get_key('the-key').get_metadata('md').should.equal('Metadatastring') @freeze_time("2012-01-01 12:00:00") @mock_s3 @@ -163,9 +173,34 @@ def test_get_all_buckets(): buckets.should.have.length_of(2) +@mock_s3 +def test_post_to_bucket(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + requests.post("https://foobar.s3.amazonaws.com/", { + 'key': 'the-key', + 'file': 'nothing' + }) + + bucket.get_key('the-key').get_contents_as_string().should.equal('nothing') + +@mock_s3 +def test_post_with_metadata_to_bucket(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + + requests.post("https://foobar.s3.amazonaws.com/", { + 'key': 'the-key', + 'file': 'nothing', + 'x-amz-meta-test': 'metadata' + }) + + bucket.get_key('the-key').get_metadata('test').should.equal('metadata') + @mock_s3 def test_bucket_method_not_implemented(): - requests.post.when.called_with("https://foobar.s3.amazonaws.com/").should.throw(NotImplementedError) + requests.patch.when.called_with("https://foobar.s3.amazonaws.com/").should.throw(NotImplementedError) @mock_s3