From be26daaff499d78b1863d9ed8501e5fd06a99c2b Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 13 Apr 2013 19:23:32 -0400 Subject: [PATCH] Fix S3 bucket list objects order and delimiters. Closes #14. --- moto/s3/models.py | 10 +++--- moto/s3/responses.py | 18 +++++++---- tests/test_s3/test_s3.py | 70 +++++++++++++++++++--------------------- 3 files changed, 51 insertions(+), 47 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 644edfb95..7912edfe9 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -87,20 +87,22 @@ class S3Backend(BaseBackend): if bucket: return bucket.keys.get(key_name) - def prefix_query(self, bucket, prefix): + def prefix_query(self, bucket, prefix, delimiter): key_results = set() folder_results = set() if prefix: for key_name, key in bucket.keys.iteritems(): if key_name.startswith(prefix): - if '/' in key_name.lstrip(prefix): - key_without_prefix = key_name.lstrip(prefix).split("/")[0] + if delimiter and '/' in key_name.lstrip(prefix): + # If delimiter, we need to split out folder_results + key_without_prefix = "{}/".format(key_name.lstrip(prefix).split("/")[0]) folder_results.add("{}{}".format(prefix, key_without_prefix)) else: key_results.add(key) else: for key_name, key in bucket.keys.iteritems(): - if '/' in key_name: + if delimiter and '/' in key_name: + # If delimiter, we need to split out folder_results folder_results.add(key_name.split("/")[0]) else: key_results.add(key) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index bc3669bc7..e5a2bed65 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -27,11 +27,13 @@ def bucket_response(uri, method, body, headers): bucket = s3_backend.get_bucket(bucket_name) if bucket: prefix = querystring.get('prefix', [None])[0] - result_keys, result_folders = s3_backend.prefix_query(bucket, prefix) + delimiter = querystring.get('delimiter') + result_keys, result_folders = s3_backend.prefix_query(bucket, prefix, delimiter) template = Template(S3_BUCKET_GET_RESPONSE) return template.render( bucket=bucket, prefix=prefix, + delimiter=delimiter, result_keys=result_keys, result_folders=result_folders ) @@ -128,7 +130,7 @@ S3_BUCKET_GET_RESPONSE = """ {{ bucket.name }} {{ prefix }} 1000 - / + {{ delimiter }} false {% for key in result_keys %} @@ -144,11 +146,13 @@ S3_BUCKET_GET_RESPONSE = """ STANDARD {% endfor %} - {% for folder in result_folders %} - - {{ folder }} - - {% endfor %} + {% if delimiter %} + {% for folder in result_folders %} + + {{ folder }} + + {% endfor %} + {% endif %} """ S3_BUCKET_CREATE_RESPONSE = """ diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index a12ea9c30..a68e511e3 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -108,42 +108,6 @@ def test_last_modified(): bucket.get_key("the-key").last_modified.should.equal('Sun, 01 Jan 2012 12:00:00 GMT') -@mock_s3 -def test_get_all_keys(): - conn = boto.connect_s3('the_key', 'the_secret') - bucket = conn.create_bucket("foobar") - key = Key(bucket) - key.key = "the-key" - key.set_contents_from_string("some value") - - key2 = Key(bucket) - key2.key = "folder/some-stuff" - key2.set_contents_from_string("some value") - - key3 = Key(bucket) - key3.key = "folder/more-folder/foobar" - key3.set_contents_from_string("some value") - - key4 = Key(bucket) - key4.key = "a-key" - key4.set_contents_from_string("some value") - - keys = bucket.get_all_keys() - keys.should.have.length_of(3) - - keys[0].name.should.equal("a-key") - keys[1].name.should.equal("the-key") - - # Prefix - keys[2].name.should.equal("folder") - - keys = bucket.get_all_keys(prefix="folder/") - keys.should.have.length_of(2) - - keys[0].name.should.equal("folder/some-stuff") - keys[1].name.should.equal("folder/more-folder") - - @mock_s3 def test_missing_bucket(): conn = boto.connect_s3('the_key', 'the_secret') @@ -218,3 +182,37 @@ def test_key_with_special_characters(): key_list = bucket.list('test_list_keys_2/', '/') keys = [x for x in key_list] keys[0].name.should.equal("test_list_keys_2/x?y") + + +@mock_s3 +def test_bucket_key_listing_order(): + conn = boto.connect_s3() + bucket = conn.create_bucket('test_bucket') + prefix = 'toplevel/' + + def store(name): + k = Key(bucket, prefix + name) + k.set_contents_from_string('somedata') + + names = ['x/key', 'y.key1', 'y.key2', 'y.key3', 'x/y/key', 'x/y/z/key'] + + for name in names: + store(name) + + delimiter = None + keys = [x.name for x in bucket.list(prefix, delimiter)] + keys.should.equal([ + 'toplevel/x/key', 'toplevel/x/y/key', 'toplevel/x/y/z/key', + 'toplevel/y.key1', 'toplevel/y.key2', 'toplevel/y.key3' + ]) + + delimiter = '/' + keys = [x.name for x in bucket.list(prefix, delimiter)] + keys.should.equal([ + 'toplevel/y.key1', 'toplevel/y.key2', 'toplevel/y.key3', 'toplevel/x/' + ]) + + # Test delimiter with no prefix + delimiter = '/' + keys = [x.name for x in bucket.list(prefix=None, delimiter=delimiter)] + keys.should.equal(['toplevel'])