From b0d89bb7b7e2c717f2e1c4d1b364458c3c9ab093 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 27 Feb 2013 01:12:11 -0500 Subject: [PATCH] fix s3 key list and missing key response --- moto/s3/models.py | 27 ++++++++++++++++++ moto/s3/responses.py | 60 +++++++++++++++++++++++++++++----------- tests/test_s3/test_s3.py | 54 +++++++++++++++++++++++++++++++++--- 3 files changed, 121 insertions(+), 20 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index c452d9a6b..f13a1761a 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -16,6 +16,10 @@ class FakeKey(object): value_md5.update(self.value) return '"{0}"'.format(value_md5.hexdigest()) + @property + def size(self): + return len(self.value) + class FakeBucket(object): def __init__(self, name): @@ -60,6 +64,29 @@ class S3Backend(BaseBackend): bucket = self.buckets[bucket_name] return bucket.keys.get(key_name) + def prefix_query(self, bucket, prefix): + key_results = set() + folder_results = set() + if prefix: + for key_name, key in bucket.keys.iteritems(): + if key_name.startswith(prefix): + if '/' in key_name.lstrip(prefix): + key_without_prefix = key_name.lstrip(prefix).split("/")[0] + folder_results.add("{}{}".format(prefix, key_without_prefix)) + else: + key_results.add(key) + else: + for key_name, key in bucket.keys.iteritems(): + if '/' in key_name: + folder_results.add(key_name.split("/")[0]) + else: + key_results.add(key) + + key_results = sorted(key_results, key=lambda key: key.name) + folder_results = [folder_name for folder_name in sorted(folder_results, key=lambda key: key)] + + return key_results, folder_results + def delete_key(self, bucket_name, key_name): bucket = self.buckets[bucket_name] return bucket.keys.pop(key_name) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index c895b8cda..65d4d5228 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1,3 +1,5 @@ +from urlparse import parse_qs + from jinja2 import Template from .models import s3_backend @@ -15,14 +17,18 @@ def all_buckets(uri, body, method): def bucket_response(uri, body, headers): hostname = uri.hostname method = uri.method + querystring = parse_qs(uri.query) bucket_name = bucket_name_from_hostname(hostname) if method == 'GET': bucket = s3_backend.get_bucket(bucket_name) if bucket: + prefix = querystring.get('prefix', [None])[0] + result_keys, result_folders = s3_backend.prefix_query(bucket, prefix) template = Template(S3_BUCKET_GET_RESPONSE) - return template.render(bucket=bucket) + return template.render(bucket=bucket, prefix=prefix, + result_keys=result_keys, result_folders=result_folders) else: return "", dict(status=404) elif method == 'PUT': @@ -58,17 +64,21 @@ def key_response(uri_info, body, headers): if method == 'GET': key = s3_backend.get_key(bucket_name, key_name) - return key.value - + if key: + return key.value + else: + return "", dict(status=404) if method == 'PUT': if 'x-amz-copy-source' in headers: # Copy key src_bucket, src_key = headers.get("x-amz-copy-source").split("/") s3_backend.copy_key(src_bucket, src_key, bucket_name, key_name) - return S3_OBJECT_COPY_RESPONSE + template = Template(S3_OBJECT_COPY_RESPONSE) + return template.render(key=src_key) if body: new_key = s3_backend.set_key(bucket_name, key_name, body) - return S3_OBJECT_RESPONSE, dict(etag=new_key.etag) + template = Template(S3_OBJECT_RESPONSE) + return template.render(key=new_key), dict(etag=new_key.etag) key = s3_backend.get_key(bucket_name, key_name) if key: return "", dict(etag=key.etag) @@ -103,15 +113,33 @@ S3_ALL_BUCKETS = """\ - {{ bucket.name }}\ - notes/\ - /\ - 1000\ - AKIAIOSFODNN7EXAMPLE\ - 2006-03-01T12:00:00.183Z\ - Iuyz3d3P0aTou39dzbqaEXAMPLE=\ - """ +S3_BUCKET_GET_RESPONSE = """ + + {{ bucket.name }} + {{ prefix }} + 1000 + / + false + {% for key in result_keys %} + + {{ key.name }} + 2006-01-01T12:00:00.000Z + {{ key.etag }} + {{ key.size }} + STANDARD + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + STANDARD + + {% endfor %} + {% for folder in result_folders %} + + {{ folder }} + + {% endfor %} + """ S3_BUCKET_CREATE_RESPONSE = """ @@ -151,14 +179,14 @@ S3_DELETE_OBJECT_SUCCESS = """ - "asdlfkdalsjfsalfkjsadlfjsdjkk" + {{ key.etag }} 2006-03-01T12:00:00.183Z """ S3_OBJECT_COPY_RESPONSE = """ - "asdfadsfdsafjsadfdafsadf" + {{ key.etag }} 2008-02-18T13:54:10.183Z """ diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index a5b54ad1f..e8ccb9075 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1,9 +1,11 @@ +import urllib2 + import boto from boto.exception import S3ResponseError from boto.s3.key import Key import requests -import sure +from sure import expect from moto import mock_s3 @@ -31,7 +33,7 @@ def test_my_model_save(): model_instance = MyModel('steve', 'is awesome') model_instance.save() - assert conn.get_bucket('mybucket').get_key('steve').get_contents_as_string() == 'is awesome' + expect(conn.get_bucket('mybucket').get_key('steve').get_contents_as_string()).should.equal('is awesome') @mock_s3 @@ -41,6 +43,14 @@ def test_missing_key(): bucket.get_key("the-key").should.equal(None) +@mock_s3 +def test_missing_key_urllib2(): + conn = boto.connect_s3('the_key', 'the_secret') + conn.create_bucket("foobar") + + urllib2.urlopen.when.called_with("http://foobar.s3.amazonaws.com/the-key").should.throw(urllib2.HTTPError) + + @mock_s3 def test_copy_key(): conn = boto.connect_s3('the_key', 'the_secret') @@ -55,6 +65,42 @@ def test_copy_key(): bucket.get_key("new-key").get_contents_as_string().should.equal("some value") +@mock_s3 +def test_get_all_keys(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket("foobar") + key = Key(bucket) + key.key = "the-key" + key.set_contents_from_string("some value") + + key2 = Key(bucket) + key2.key = "folder/some-stuff" + key2.set_contents_from_string("some value") + + key3 = Key(bucket) + key3.key = "folder/more-folder/foobar" + key3.set_contents_from_string("some value") + + key4 = Key(bucket) + key4.key = "a-key" + key4.set_contents_from_string("some value") + + keys = bucket.get_all_keys() + keys.should.have.length_of(3) + + keys[0].name.should.equal("a-key") + keys[1].name.should.equal("the-key") + + # Prefix + keys[2].name.should.equal("folder") + + keys = bucket.get_all_keys(prefix="folder/") + keys.should.have.length_of(2) + + keys[0].name.should.equal("folder/some-stuff") + keys[1].name.should.equal("folder/more-folder") + + @mock_s3 def test_missing_bucket(): conn = boto.connect_s3('the_key', 'the_secret') @@ -86,8 +132,8 @@ def test_bucket_deletion(): @mock_s3 def test_get_all_buckets(): conn = boto.connect_s3('the_key', 'the_secret') - bucket = conn.create_bucket("foobar") - bucket = conn.create_bucket("foobar2") + conn.create_bucket("foobar") + conn.create_bucket("foobar2") buckets = conn.get_all_buckets() buckets.should.have.length_of(2)