Move S3 storage to SpooledTemporaryFile

moto/s3/models.py

@@ -8,6 +8,7 @@ import itertools
 import codecs
 import random
 import string
+import tempfile
 
 import six
 
@@ -21,6 +22,7 @@ from .utils import clean_key_name, _VersionedKeyStore
 UPLOAD_ID_BYTES = 43
 UPLOAD_PART_MIN_SIZE = 5242880
 STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]
+DEFAULT_KEY_BUFFER_SIZE = 2 ** 24
 
 
 class FakeDeleteMarker(BaseModel):
@@ -42,9 +44,9 @@ class FakeDeleteMarker(BaseModel):
 
 class FakeKey(BaseModel):
 
-    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0):
+    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0,
+                 max_buffer_size=DEFAULT_KEY_BUFFER_SIZE):
         self.name = name
-        self.value = value
         self.last_modified = datetime.datetime.utcnow()
         self.acl = get_canned_acl('private')
         self.website_redirect_location = None
@@ -56,10 +58,24 @@ class FakeKey(BaseModel):
         self._is_versioned = is_versioned
         self._tagging = FakeTagging()
 
+        self.value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
+        self.value = value
+
     @property
     def version_id(self):
         return self._version_id
 
+    @property
+    def value(self):
+        self.value_buffer.seek(0)
+        return self.value_buffer.read()
+
+    @value.setter
+    def value(self, new_value):
+        self.value_buffer.seek(0)
+        self.value_buffer.truncate()
+        self.value_buffer.write(new_value)
+
     def copy(self, new_name=None):
         r = copy.deepcopy(self)
         if new_name is not None:
@@ -83,7 +99,9 @@ class FakeKey(BaseModel):
         self.acl = acl
 
     def append_to_value(self, value):
-        self.value += value
+        self.value_buffer.seek(0, os.SEEK_END)
+        self.value_buffer.write(value)
+
         self.last_modified = datetime.datetime.utcnow()
         self._etag = None  # must recalculate etag
         if self._is_versioned:
@@ -101,11 +119,14 @@ class FakeKey(BaseModel):
     def etag(self):
         if self._etag is None:
             value_md5 = hashlib.md5()
-            if isinstance(self.value, six.text_type):
-                value = self.value.encode("utf-8")
-            else:
-                value = self.value
-            value_md5.update(value)
+
+            self.value_buffer.seek(0)
+            while True:
+                block = self.value_buffer.read(DEFAULT_KEY_BUFFER_SIZE)
+                if not block:
+                    break
+                value_md5.update(block)
+
             self._etag = value_md5.hexdigest()
         return '"{0}"'.format(self._etag)
 
@@ -132,7 +153,7 @@ class FakeKey(BaseModel):
         res = {
             'ETag': self.etag,
             'last-modified': self.last_modified_RFC1123,
-            'content-length': str(len(self.value)),
+            'content-length': str(self.size),
         }
         if self._storage_class != 'STANDARD':
             res['x-amz-storage-class'] = self._storage_class
@@ -150,7 +171,8 @@ class FakeKey(BaseModel):
 
     @property
     def size(self):
-        return len(self.value)
+        self.value_buffer.seek(0, os.SEEK_END)
+        return self.value_buffer.tell()
 
     @property
     def storage_class(self):

tests/test_s3/test_s3.py

@@ -524,7 +524,7 @@ def test_post_to_bucket():
 
     requests.post("https://foobar.s3.amazonaws.com/", {
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     bucket.get_key('the-key').get_contents_as_string().should.equal(b'nothing')
@@ -538,7 +538,7 @@ def test_post_with_metadata_to_bucket():
 
     requests.post("https://foobar.s3.amazonaws.com/", {
         'key': 'the-key',
-        'file': 'nothing',
+        'file': b'nothing',
         'x-amz-meta-test': 'metadata'
     })
 

tests/test_s3/test_server.py

@@ -72,7 +72,7 @@ def test_s3_server_post_to_bucket():
 
     test_client.post('/', "https://tester.localhost:5000/", data={
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     res = test_client.get('/the-key', 'http://tester.localhost:5000/')

tests/test_s3bucket_path/test_bucket_path_server.py

@@ -73,7 +73,7 @@ def test_s3_server_post_to_bucket():
 
     test_client.post('/foobar2', "https://localhost:5000/", data={
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     res = test_client.get('/foobar2/the-key', 'http://localhost:5000/')
@@ -89,7 +89,7 @@ def test_s3_server_put_ipv6():
 
     test_client.post('/foobar2', "https://[::]:5000/", data={
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     res = test_client.get('/foobar2/the-key', 'http://[::]:5000/')
@@ -105,7 +105,7 @@ def test_s3_server_put_ipv4():
 
     test_client.post('/foobar2', "https://127.0.0.1:5000/", data={
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     res = test_client.get('/foobar2/the-key', 'http://127.0.0.1:5000/')

tests/test_s3bucket_path/test_s3bucket_path.py

@@ -198,7 +198,7 @@ def test_post_to_bucket():
 
     requests.post("https://s3.amazonaws.com/foobar", {
         'key': 'the-key',
-        'file': 'nothing'
+        'file': b'nothing'
     })
 
     bucket.get_key('the-key').get_contents_as_string().should.equal(b'nothing')
@@ -212,7 +212,7 @@ def test_post_with_metadata_to_bucket():
 
     requests.post("https://s3.amazonaws.com/foobar", {
         'key': 'the-key',
-        'file': 'nothing',
+        'file': b'nothing',
         'x-amz-meta-test': 'metadata'
     })