From dbf1e64d4406e70a3d96e9c54cafebe0f252bfcb Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 10 Feb 2015 17:28:18 +0200 Subject: [PATCH 1/6] support ranged get in S3 --- moto/s3/responses.py | 9 +++++++-- tests/test_s3/test_s3.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 750bf68ba..6ba401b90 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -234,11 +234,16 @@ class ResponseObject(_TemplateEnvironmentMixin): except MissingBucket: return 404, headers, "" + begin = 0 + end = None + if 'range' in request.headers: + begin, end = map(int, request.headers.get('range').split('-')) + if isinstance(response, six.string_types): - return 200, headers, response + return 200, headers, response[begin:end] else: status_code, headers, response_content = response - return status_code, headers, response_content + return status_code, headers, response_content[begin:end] def _key_response(self, request, full_url, headers): parsed_url = urlparse(full_url) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 1643eaaff..ecfeb23c7 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -692,3 +692,13 @@ def test_bucket_location(): conn = boto.s3.connect_to_region("us-west-2") bucket = conn.create_bucket('mybucket') bucket.get_location().should.equal("us-west-2") + + +@mock_s3 +def test_ranged_get(): + conn = boto.connect_s3() + bucket = conn.create_bucket('mybucket') + key = Key(bucket) + key.key = 'bigkey' + key.set_contents_from_string('0' * 50 + '1' * 50) + key.get_contents_as_string(headers={'Range': '45-55'}).should.equal('0' * 5 + '1' * 5) From 9efd12c43cb17dabdd678dc5532daafb83fdcd6e Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 10 Feb 2015 17:55:44 +0200 Subject: [PATCH 2/6] support more range specifiers in ranged gets --- moto/s3/responses.py | 5 ++++- tests/test_s3/test_s3.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 6ba401b90..420b10ce7 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -237,7 +237,10 @@ class ResponseObject(_TemplateEnvironmentMixin): begin = 0 end = None if 'range' in request.headers: - begin, end = map(int, request.headers.get('range').split('-')) + _, rspec = request.headers.get('range').split('=') + if ',' in rspec: + raise NotImplementedError("Multiple range specifiers not supported") + begin, end = map(lambda i: int(i) if i else None, rspec.split('-')) if isinstance(response, six.string_types): return 200, headers, response[begin:end] diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index ecfeb23c7..1b4077226 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -701,4 +701,6 @@ def test_ranged_get(): key = Key(bucket) key.key = 'bigkey' key.set_contents_from_string('0' * 50 + '1' * 50) - key.get_contents_as_string(headers={'Range': '45-55'}).should.equal('0' * 5 + '1' * 5) + key.get_contents_as_string(headers={'Range': 'bytes=45-55'}).should.equal(b'0' * 5 + b'1' * 5) + key.get_contents_as_string(headers={'Range': 'bytes=45-'}).should.equal(b'0' * 5 + b'1' * 50) + key.get_contents_as_string(headers={'Range': 'bytes=-55'}).should.equal(b'0' * 50 + b'1' * 5) From 418a6a118e246fd182497027acd5b5c09afde8de Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 10 Feb 2015 19:14:47 +0200 Subject: [PATCH 3/6] return proper status codes in range get, fix suffix range --- moto/s3/responses.py | 35 +++++++++++++++++++++++++---------- tests/test_s3/test_s3.py | 2 +- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 420b10ce7..4d0ae20a4 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -228,25 +228,40 @@ class ResponseObject(_TemplateEnvironmentMixin): return 200, headers, template.render(deleted=deleted_names, delete_errors=error_names) + def _handle_range_header(self, request, headers, response_content): + length = len(response_content) + _, rspec = request.headers.get('range').split('=') + if ',' in rspec: + raise NotImplementedError( + "Multiple range specifiers not supported") + toint = lambda i: int(i) if i else None + begin, end = map(toint, rspec.split('-')) + if begin is not None: # byte range + end = end or length + elif end is not None: # suffix byte range + begin = length - end + end = length + else: + return 400, headers, "" + if begin < 0 or end > length or begin > min(end, length): + return 416, headers, "" + return 206, headers, response_content[begin:end] + def key_response(self, request, full_url, headers): try: response = self._key_response(request, full_url, headers) except MissingBucket: return 404, headers, "" - begin = 0 - end = None - if 'range' in request.headers: - _, rspec = request.headers.get('range').split('=') - if ',' in rspec: - raise NotImplementedError("Multiple range specifiers not supported") - begin, end = map(lambda i: int(i) if i else None, rspec.split('-')) - if isinstance(response, six.string_types): - return 200, headers, response[begin:end] + status_code = 200 + response_content = response else: status_code, headers, response_content = response - return status_code, headers, response_content[begin:end] + + if status_code == 200 and 'range' in request.headers: + return self._handle_range_header(request, headers, response_content) + return status_code, headers, response_content def _key_response(self, request, full_url, headers): parsed_url = urlparse(full_url) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 1b4077226..5e7d0e096 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -703,4 +703,4 @@ def test_ranged_get(): key.set_contents_from_string('0' * 50 + '1' * 50) key.get_contents_as_string(headers={'Range': 'bytes=45-55'}).should.equal(b'0' * 5 + b'1' * 5) key.get_contents_as_string(headers={'Range': 'bytes=45-'}).should.equal(b'0' * 5 + b'1' * 50) - key.get_contents_as_string(headers={'Range': 'bytes=-55'}).should.equal(b'0' * 50 + b'1' * 5) + key.get_contents_as_string(headers={'Range': 'bytes=-55'}).should.equal(b'0' * 5 + b'1' * 50) From 261328d449b186751f82a7265b8a33d113eea914 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 10 Feb 2015 19:15:10 +0200 Subject: [PATCH 4/6] set content-range header so boto knows this is a ranged response --- moto/s3/responses.py | 2 ++ tests/test_s3/test_s3.py | 1 + 2 files changed, 3 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 4d0ae20a4..a0750e5ac 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -245,6 +245,8 @@ class ResponseObject(_TemplateEnvironmentMixin): return 400, headers, "" if begin < 0 or end > length or begin > min(end, length): return 416, headers, "" + headers['content-range'] = "bytes {0}-{1}/{2}".format( + begin, end, length) return 206, headers, response_content[begin:end] def key_response(self, request, full_url, headers): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 5e7d0e096..3b30ef2e7 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -704,3 +704,4 @@ def test_ranged_get(): key.get_contents_as_string(headers={'Range': 'bytes=45-55'}).should.equal(b'0' * 5 + b'1' * 5) key.get_contents_as_string(headers={'Range': 'bytes=45-'}).should.equal(b'0' * 5 + b'1' * 50) key.get_contents_as_string(headers={'Range': 'bytes=-55'}).should.equal(b'0' * 5 + b'1' * 50) + key.size.should.equal(100) From e1163e5223a57afadb7ff04222da735455cef2b2 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Tue, 10 Feb 2015 19:43:24 +0200 Subject: [PATCH 5/6] fix some boundary conditions --- moto/s3/responses.py | 9 +++++---- tests/test_s3/test_s3.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index a0750e5ac..5cd6fb9b0 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -230,6 +230,7 @@ class ResponseObject(_TemplateEnvironmentMixin): def _handle_range_header(self, request, headers, response_content): length = len(response_content) + last = length - 1 _, rspec = request.headers.get('range').split('=') if ',' in rspec: raise NotImplementedError( @@ -237,17 +238,17 @@ class ResponseObject(_TemplateEnvironmentMixin): toint = lambda i: int(i) if i else None begin, end = map(toint, rspec.split('-')) if begin is not None: # byte range - end = end or length + end = last if end is None else end elif end is not None: # suffix byte range begin = length - end - end = length + end = last else: return 400, headers, "" - if begin < 0 or end > length or begin > min(end, length): + if begin < 0 or end > length or begin > min(end, last): return 416, headers, "" headers['content-range'] = "bytes {0}-{1}/{2}".format( begin, end, length) - return 206, headers, response_content[begin:end] + return 206, headers, response_content[begin:end + 1] def key_response(self, request, full_url, headers): try: diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 3b30ef2e7..e1a5a0355 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -700,8 +700,13 @@ def test_ranged_get(): bucket = conn.create_bucket('mybucket') key = Key(bucket) key.key = 'bigkey' - key.set_contents_from_string('0' * 50 + '1' * 50) - key.get_contents_as_string(headers={'Range': 'bytes=45-55'}).should.equal(b'0' * 5 + b'1' * 5) - key.get_contents_as_string(headers={'Range': 'bytes=45-'}).should.equal(b'0' * 5 + b'1' * 50) - key.get_contents_as_string(headers={'Range': 'bytes=-55'}).should.equal(b'0' * 5 + b'1' * 50) + rep = "0123456789" + key.set_contents_from_string(rep * 10) + key.get_contents_as_string(headers={'Range': 'bytes=0-'}).should.equal(rep * 10) + key.get_contents_as_string(headers={'Range': 'bytes=0-99'}).should.equal(rep * 10) + key.get_contents_as_string(headers={'Range': 'bytes=0-0'}).should.equal(b'0') + key.get_contents_as_string(headers={'Range': 'bytes=99-99'}).should.equal(b'9') + key.get_contents_as_string(headers={'Range': 'bytes=50-54'}).should.equal(rep[:5]) + key.get_contents_as_string(headers={'Range': 'bytes=50-'}).should.equal(rep * 5) + key.get_contents_as_string(headers={'Range': 'bytes=-60'}).should.equal(rep * 6) key.size.should.equal(100) From b2904fe84918df1e4de1375610fff1784220cee3 Mon Sep 17 00:00:00 2001 From: Konstantinos Koukopoulos Date: Wed, 11 Feb 2015 11:08:00 +0200 Subject: [PATCH 6/6] add byte specifier for py3 --- tests/test_s3/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index e1a5a0355..e879f3379 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -700,7 +700,7 @@ def test_ranged_get(): bucket = conn.create_bucket('mybucket') key = Key(bucket) key.key = 'bigkey' - rep = "0123456789" + rep = b"0123456789" key.set_contents_from_string(rep * 10) key.get_contents_as_string(headers={'Range': 'bytes=0-'}).should.equal(rep * 10) key.get_contents_as_string(headers={'Range': 'bytes=0-99'}).should.equal(rep * 10)