From eb6515cf50878b18081ecf0afe944a36c1bdda5f Mon Sep 17 00:00:00 2001
From: benediktbrandt <30300995+benediktbrandt@users.noreply.github.com>
Date: Wed, 7 Jul 2021 11:38:50 -0400
Subject: [PATCH] Reduce default value for DEFAULT_KEY_BUFFER_SIZE (#4003)

* - introduce environment variable for DEFAULT_KEY_BUFFER_SIZE

* - prefix env variable with MOTO_S3 to avoid env variable conflicts

* - reduce the DEFAULT_KEY_BUFFER_SIZE to be less than the S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads

* - black formatting

* - fix formatting

* - fix missing import
---
 moto/s3/models.py        |  5 ++---
 moto/settings.py         |  9 ++++++++-
 tests/test_s3/test_s3.py | 15 ++++++++++-----
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/moto/s3/models.py b/moto/s3/models.py
index 17ecff50b..5c976195f 100644
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -45,12 +45,11 @@ from .exceptions import (
 )
 from .cloud_formation import cfn_to_api_encryption, is_replacement_update
 from .utils import clean_key_name, _VersionedKeyStore
-from ..settings import get_s3_default_key_buffer_size
+from ..settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
 
 MAX_BUCKET_NAME_LENGTH = 63
 MIN_BUCKET_NAME_LENGTH = 3
 UPLOAD_ID_BYTES = 43
-UPLOAD_PART_MIN_SIZE = 5242880
 STORAGE_CLASS = [
     "STANDARD",
     "REDUCED_REDUNDANCY",
@@ -320,7 +319,7 @@ class FakeMultipart(BaseModel):
                 etag = etag.replace('"', "")
             if part is None or part_etag != etag:
                 raise InvalidPart()
-            if last is not None and last.contentsize < UPLOAD_PART_MIN_SIZE:
+            if last is not None and last.contentsize < S3_UPLOAD_PART_MIN_SIZE:
                 raise EntityTooSmall()
             md5s.extend(decode_hex(part_etag)[0])
             total.extend(part.value)
diff --git a/moto/settings.py b/moto/settings.py
index c753ac346..696c5d83b 100644
--- a/moto/settings.py
+++ b/moto/settings.py
@@ -23,5 +23,12 @@ def get_sf_execution_history_type():
     return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS")
 
 
+S3_UPLOAD_PART_MIN_SIZE = 5242880
+
+
 def get_s3_default_key_buffer_size():
-    return int(os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", 16 * 1024 * 1024))
+    return int(
+        os.environ.get(
+            "MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", S3_UPLOAD_PART_MIN_SIZE - 1024
+        )
+    )
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
index a7b362cb4..f59520366 100644
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -39,10 +39,10 @@ from moto import settings, mock_s3, mock_s3_deprecated, mock_config
 import moto.s3.models as s3model
 from moto.core.exceptions import InvalidNextTokenException
 from moto.core.utils import py2_strip_unicode_keys
-from moto.settings import get_s3_default_key_buffer_size
+from moto.settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
 
 if settings.TEST_SERVER_MODE:
-    REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE
+    REDUCED_PART_SIZE = S3_UPLOAD_PART_MIN_SIZE
     EXPECTED_ETAG = '"140f92a6df9f9e415f74a1463bcee9bb-2"'
 else:
     REDUCED_PART_SIZE = 256
@@ -53,15 +53,15 @@ def reduced_min_part_size(f):
     """speed up tests by temporarily making the multipart minimum part size
     small
     """
-    orig_size = s3model.UPLOAD_PART_MIN_SIZE
+    orig_size = S3_UPLOAD_PART_MIN_SIZE
 
     @wraps(f)
     def wrapped(*args, **kwargs):
         try:
-            s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
+            s3model.S3_UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
             return f(*args, **kwargs)
         finally:
-            s3model.UPLOAD_PART_MIN_SIZE = orig_size
+            s3model.S3_UPLOAD_PART_MIN_SIZE = orig_size
 
     return wrapped
 
@@ -1143,6 +1143,11 @@ def test_default_key_buffer_size():
     fk = models.FakeKey("a", os.urandom(3))  # 3 byte string
     assert fk._value_buffer._rolled == True
 
+    # if no MOTO_S3_DEFAULT_KEY_BUFFER_SIZE env variable is present the buffer size should be less than
+    # S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads
+    del os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"]
+    assert get_s3_default_key_buffer_size() < S3_UPLOAD_PART_MIN_SIZE
+
     # restore original environment variable content
     if original_default_key_buffer_size:
         os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"] = original_default_key_buffer_size