Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce default value for DEFAULT_KEY_BUFFER_SIZE (#4003)

Browse Source 
* - introduce environment variable for DEFAULT_KEY_BUFFER_SIZE

* - prefix env variable with MOTO_S3 to avoid env variable conflicts

* - reduce the DEFAULT_KEY_BUFFER_SIZE to be less than the S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads

* - black formatting

* - fix formatting

* - fix missing import
This commit is contained in:
benediktbrandt 2021-07-07 11:38:50 -04:00 committed by GitHub
parent 9c256ac75e
commit eb6515cf50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
moto/s3/models.py
View File

@ -45,12 +45,11 @@ from .exceptions import (
)
from .cloud_formation import cfn_to_api_encryption, is_replacement_update
from .utils import clean_key_name, _VersionedKeyStore
from ..settings import get_s3_default_key_buffer_size
from ..settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
MAX_BUCKET_NAME_LENGTH = 63
MIN_BUCKET_NAME_LENGTH = 3
UPLOAD_ID_BYTES = 43
UPLOAD_PART_MIN_SIZE = 5242880
STORAGE_CLASS = [
"STANDARD",
"REDUCED_REDUNDANCY",
@ -320,7 +319,7 @@ class FakeMultipart(BaseModel):
etag = etag.replace('"', "")
if part is None or part_etag != etag:
raise InvalidPart()
if last is not None and last.contentsize < UPLOAD_PART_MIN_SIZE:
if last is not None and last.contentsize < S3_UPLOAD_PART_MIN_SIZE:
raise EntityTooSmall()
md5s.extend(decode_hex(part_etag)[0])
total.extend(part.value)

9
moto/settings.py
View File

@ -23,5 +23,12 @@ def get_sf_execution_history_type():
return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS")
S3_UPLOAD_PART_MIN_SIZE = 5242880
def get_s3_default_key_buffer_size():
return int(os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", 16 * 1024 * 1024))
return int(
os.environ.get(
"MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", S3_UPLOAD_PART_MIN_SIZE - 1024
)
)

15
tests/test_s3/test_s3.py
View File

@ -39,10 +39,10 @@ from moto import settings, mock_s3, mock_s3_deprecated, mock_config
import moto.s3.models as s3model
from moto.core.exceptions import InvalidNextTokenException
from moto.core.utils import py2_strip_unicode_keys
from moto.settings import get_s3_default_key_buffer_size
from moto.settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
if settings.TEST_SERVER_MODE:
REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE
REDUCED_PART_SIZE = S3_UPLOAD_PART_MIN_SIZE
EXPECTED_ETAG = '"140f92a6df9f9e415f74a1463bcee9bb-2"'
else:
REDUCED_PART_SIZE = 256
@ -53,15 +53,15 @@ def reduced_min_part_size(f):
"""speed up tests by temporarily making the multipart minimum part size
small
"""
orig_size = s3model.UPLOAD_PART_MIN_SIZE
orig_size = S3_UPLOAD_PART_MIN_SIZE
@wraps(f)
def wrapped(*args, **kwargs):
try:
s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
s3model.S3_UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
return f(*args, **kwargs)
finally:
s3model.UPLOAD_PART_MIN_SIZE = orig_size
s3model.S3_UPLOAD_PART_MIN_SIZE = orig_size
return wrapped
@ -1143,6 +1143,11 @@ def test_default_key_buffer_size():
fk = models.FakeKey("a", os.urandom(3)) # 3 byte string
assert fk._value_buffer._rolled == True
# if no MOTO_S3_DEFAULT_KEY_BUFFER_SIZE env variable is present the buffer size should be less than
# S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads
del os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"]
assert get_s3_default_key_buffer_size() < S3_UPLOAD_PART_MIN_SIZE
# restore original environment variable content
if original_default_key_buffer_size:
os.environ["MOTO_S3_DEFAULT_KEY_BUFFER_SIZE"] = original_default_key_buffer_size