From b0a280bde2c7a722c8d7e1b278c3b458781ecf6d Mon Sep 17 00:00:00 2001 From: Diego Argueta Date: Tue, 18 Dec 2018 14:08:37 -0800 Subject: [PATCH] Move S3 storage to SpooledTemporaryFile --- moto/s3/models.py | 42 ++++++++++++++----- tests/test_s3/test_s3.py | 4 +- tests/test_s3/test_server.py | 2 +- .../test_bucket_path_server.py | 6 +-- .../test_s3bucket_path/test_s3bucket_path.py | 4 +- 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index bb4d7848c..4ce2afb34 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -8,6 +8,7 @@ import itertools import codecs import random import string +import tempfile import six @@ -21,6 +22,7 @@ from .utils import clean_key_name, _VersionedKeyStore UPLOAD_ID_BYTES = 43 UPLOAD_PART_MIN_SIZE = 5242880 STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"] +DEFAULT_KEY_BUFFER_SIZE = 2 ** 24 class FakeDeleteMarker(BaseModel): @@ -42,9 +44,9 @@ class FakeDeleteMarker(BaseModel): class FakeKey(BaseModel): - def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0): + def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0, + max_buffer_size=DEFAULT_KEY_BUFFER_SIZE): self.name = name - self.value = value self.last_modified = datetime.datetime.utcnow() self.acl = get_canned_acl('private') self.website_redirect_location = None @@ -56,10 +58,24 @@ class FakeKey(BaseModel): self._is_versioned = is_versioned self._tagging = FakeTagging() + self.value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size) + self.value = value + @property def version_id(self): return self._version_id + @property + def value(self): + self.value_buffer.seek(0) + return self.value_buffer.read() + + @value.setter + def value(self, new_value): + self.value_buffer.seek(0) + self.value_buffer.truncate() + self.value_buffer.write(new_value) + def copy(self, new_name=None): r = copy.deepcopy(self) if new_name is not None: @@ -83,7 +99,9 @@ class FakeKey(BaseModel): self.acl = acl def append_to_value(self, value): - self.value += value + self.value_buffer.seek(0, os.SEEK_END) + self.value_buffer.write(value) + self.last_modified = datetime.datetime.utcnow() self._etag = None # must recalculate etag if self._is_versioned: @@ -101,11 +119,14 @@ class FakeKey(BaseModel): def etag(self): if self._etag is None: value_md5 = hashlib.md5() - if isinstance(self.value, six.text_type): - value = self.value.encode("utf-8") - else: - value = self.value - value_md5.update(value) + + self.value_buffer.seek(0) + while True: + block = self.value_buffer.read(DEFAULT_KEY_BUFFER_SIZE) + if not block: + break + value_md5.update(block) + self._etag = value_md5.hexdigest() return '"{0}"'.format(self._etag) @@ -132,7 +153,7 @@ class FakeKey(BaseModel): res = { 'ETag': self.etag, 'last-modified': self.last_modified_RFC1123, - 'content-length': str(len(self.value)), + 'content-length': str(self.size), } if self._storage_class != 'STANDARD': res['x-amz-storage-class'] = self._storage_class @@ -150,7 +171,8 @@ class FakeKey(BaseModel): @property def size(self): - return len(self.value) + self.value_buffer.seek(0, os.SEEK_END) + return self.value_buffer.tell() @property def storage_class(self): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb6..07137bcfa 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -524,7 +524,7 @@ def test_post_to_bucket(): requests.post("https://foobar.s3.amazonaws.com/", { 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) bucket.get_key('the-key').get_contents_as_string().should.equal(b'nothing') @@ -538,7 +538,7 @@ def test_post_with_metadata_to_bucket(): requests.post("https://foobar.s3.amazonaws.com/", { 'key': 'the-key', - 'file': 'nothing', + 'file': b'nothing', 'x-amz-meta-test': 'metadata' }) diff --git a/tests/test_s3/test_server.py b/tests/test_s3/test_server.py index 9c8252a04..934dd5a42 100644 --- a/tests/test_s3/test_server.py +++ b/tests/test_s3/test_server.py @@ -72,7 +72,7 @@ def test_s3_server_post_to_bucket(): test_client.post('/', "https://tester.localhost:5000/", data={ 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) res = test_client.get('/the-key', 'http://tester.localhost:5000/') diff --git a/tests/test_s3bucket_path/test_bucket_path_server.py b/tests/test_s3bucket_path/test_bucket_path_server.py index 434110e87..604adc289 100644 --- a/tests/test_s3bucket_path/test_bucket_path_server.py +++ b/tests/test_s3bucket_path/test_bucket_path_server.py @@ -73,7 +73,7 @@ def test_s3_server_post_to_bucket(): test_client.post('/foobar2', "https://localhost:5000/", data={ 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) res = test_client.get('/foobar2/the-key', 'http://localhost:5000/') @@ -89,7 +89,7 @@ def test_s3_server_put_ipv6(): test_client.post('/foobar2', "https://[::]:5000/", data={ 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) res = test_client.get('/foobar2/the-key', 'http://[::]:5000/') @@ -105,7 +105,7 @@ def test_s3_server_put_ipv4(): test_client.post('/foobar2', "https://127.0.0.1:5000/", data={ 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) res = test_client.get('/foobar2/the-key', 'http://127.0.0.1:5000/') diff --git a/tests/test_s3bucket_path/test_s3bucket_path.py b/tests/test_s3bucket_path/test_s3bucket_path.py index 21d786c61..58ae4db6f 100644 --- a/tests/test_s3bucket_path/test_s3bucket_path.py +++ b/tests/test_s3bucket_path/test_s3bucket_path.py @@ -198,7 +198,7 @@ def test_post_to_bucket(): requests.post("https://s3.amazonaws.com/foobar", { 'key': 'the-key', - 'file': 'nothing' + 'file': b'nothing' }) bucket.get_key('the-key').get_contents_as_string().should.equal(b'nothing') @@ -212,7 +212,7 @@ def test_post_with_metadata_to_bucket(): requests.post("https://s3.amazonaws.com/foobar", { 'key': 'the-key', - 'file': 'nothing', + 'file': b'nothing', 'x-amz-meta-test': 'metadata' })