Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce default value for DEFAULT_KEY_BUFFER_SIZE (#4003)

Browse Source 
* - introduce environment variable for DEFAULT_KEY_BUFFER_SIZE

* - prefix env variable with MOTO_S3 to avoid env variable conflicts

* - reduce the DEFAULT_KEY_BUFFER_SIZE to be less than the S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads

* - black formatting

* - fix formatting

* - fix missing import
This commit is contained in:
benediktbrandt 2021-07-07 11:38:50 -04:00 committed by GitHub
parent 9c256ac75e
commit eb6515cf50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
moto/s3/models.py
View File

@ -45,12 +45,11 @@ from .exceptions import (
) )
from .cloud_formation import cfn_to_api_encryption, is_replacement_update from .cloud_formation import cfn_to_api_encryption, is_replacement_update
from .utils import clean_key_name, _VersionedKeyStore from .utils import clean_key_name, _VersionedKeyStore
from ..settings import get_s3_default_key_buffer_size from ..settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
MAX_BUCKET_NAME_LENGTH = 63 MAX_BUCKET_NAME_LENGTH = 63
MIN_BUCKET_NAME_LENGTH = 3 MIN_BUCKET_NAME_LENGTH = 3
UPLOAD_ID_BYTES = 43 UPLOAD_ID_BYTES = 43
UPLOAD_PART_MIN_SIZE = 5242880
STORAGE_CLASS = [ STORAGE_CLASS = [
"STANDARD", "STANDARD",
"REDUCED_REDUNDANCY", "REDUCED_REDUNDANCY",
@ -320,7 +319,7 @@ class FakeMultipart(BaseModel):
etag = etag.replace('"', "") etag = etag.replace('"', "")
if part is None or part_etag != etag: if part is None or part_etag != etag:
raise InvalidPart() raise InvalidPart()
if last is not None and last.contentsize < UPLOAD_PART_MIN_SIZE: if last is not None and last.contentsize < S3_UPLOAD_PART_MIN_SIZE:
raise EntityTooSmall() raise EntityTooSmall()
md5s.extend(decode_hex(part_etag)[0]) md5s.extend(decode_hex(part_etag)[0])
total.extend(part.value) total.extend(part.value)

9
moto/settings.py
View File

@ -23,5 +23,12 @@ def get_sf_execution_history_type():
return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS") return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS")
S3_UPLOAD_PART_MIN_SIZE = 5242880
def get_s3_default_key_buffer_size(): def get_s3_default_key_buffer_size():
return int(os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", 16 * 1024 * 1024)) return int(
os.environ.get(
"MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", S3_UPLOAD_PART_MIN_SIZE - 1024
)
)

15
tests/test_s3/test_s3.py
View File

@ -39,10 +39,10 @@ from moto import settings, mock_s3, mock_s3_deprecated, mock_config
import moto.s3.models as s3model import moto.s3.models as s3model
from moto.core.exceptions import InvalidNextTokenException from moto.core.exceptions import InvalidNextTokenException
from moto.core.utils import py2_strip_unicode_keys from moto.core.utils import py2_strip_unicode_keys
from moto.settings import get_s3_default_key_buffer_size from moto.settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
if settings.TEST_SERVER_MODE: if settings.TEST_SERVER_MODE:
REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE REDUCED_PART_SIZE = S3_UPLOAD_PART_MIN_SIZE
EXPECTED_ETAG = '"140f92a6df9f9e415f74a1463bcee9bb-2"' EXPECTED_ETAG = '"140f92a6df9f9e415f74a1463bcee9bb-2"'
else: else:
REDUCED_PART_SIZE = 256 REDUCED_PART_SIZE = 256
@ -53,15 +53,15 @@ def reduced_min_part_size(f):
"""speed up tests by temporarily making the multipart minimum part size """speed up tests by temporarily making the multipart minimum part size
small small
""" """
orig_size = s3model.UPLOAD_PART_MIN_SIZE orig_size = S3_UPLOAD_PART_MIN_SIZE
@wraps(f) @wraps(f)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
try: try:
s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE s3model.S3_UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
return f(*args, **kwargs) return f(*args, **kwargs)
finally: finally:
s3model.UPLOAD_PART_MIN_SIZE = orig_size s3model.S3_UPLOAD_PART_MIN_SIZE = orig_size
return wrapped return wrapped
@ -1143,6 +1143,11 @@ def test_default_key_buffer_size():
fk = models.FakeKey("a", os.urandom(3)) # 3 byte string fk = models.FakeKey("a", os.urandom(3)) # 3 byte string
assert fk._value_buffer._rolled == True assert fk._value_buffer._rolled == True
# if no MOTO_S3_DEFAULT_KEY_BUFFER_SIZE env variable is present the buffer size should be less than
# S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads
del os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"]
assert get_s3_default_key_buffer_size() < S3_UPLOAD_PART_MIN_SIZE
# restore original environment variable content # restore original environment variable content
if original_default_key_buffer_size: if original_default_key_buffer_size:
os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"] = original_default_key_buffer_size os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"] = original_default_key_buffer_size