From e82e1e3f397cd610d3ed0316c37325cdfe55926b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 21 Mar 2020 12:20:09 +0000 Subject: [PATCH 1/6] DynamoDB - Add 1MB item size check --- moto/dynamodb2/models.py | 11 ++++++++ tests/test_dynamodb2/test_dynamodb.py | 38 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 54dccd56d..a35eded61 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -285,6 +285,9 @@ class Item(BaseModel): def __repr__(self): return "Item: {0}".format(self.to_json()) + def size(self): + return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + def to_json(self): attributes = {} for attribute_key, attribute in self.attrs.items(): @@ -1123,6 +1126,14 @@ class Table(BaseModel): break last_evaluated_key = None + size_limit = 1000000 # DynamoDB has a 1MB size limit + item_size = sum([res.size() for res in results]) + if item_size > size_limit: + item_size = idx = 0 + while item_size + results[idx].size() < size_limit: + item_size += results[idx].size() + idx += 1 + limit = min(limit, idx) if limit else idx if limit and len(results) > limit: results = results[:limit] last_evaluated_key = {self.hash_key_attr: results[-1].hash_key} diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 062208863..daae79232 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4132,3 +4132,41 @@ def test_gsi_verify_negative_number_order(): [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( [-0.7, -0.6, 0.7] ) + + +@mock_dynamodb2 +def test_dynamodb_max_1mb_limit(): + ddb = boto3.resource("dynamodb", region_name="eu-west-1") + + table_name = "populated-mock-table" + table = ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "SORT"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + # Populate the table + items = [ + { + "partition_key": "partition_key_val", # size=30 + "sort_key": "sort_key_value____" + str(i), # size=30 + } + for i in range(10000, 29999) + ] + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + response = table.query( + KeyConditionExpression=Key("partition_key").eq("partition_key_val") + ) + # We shouldn't get everything back - the total result set is well over 1MB + assert response["Count"] < len(items) + response["LastEvaluatedKey"].shouldnt.be(None) From 1e1fe3ee4bffb5470795d8aab16fa3de1145f5a6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:13 +0100 Subject: [PATCH 2/6] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a35eded61..de2a06fd4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -286,7 +286,7 @@ class Item(BaseModel): return "Item: {0}".format(self.to_json()) def size(self): - return sum([bytesize(key) + value.size() for key, value in self.attrs.items()]) + return sum(bytesize(key) + value.size() for key, value in self.attrs.items()) def to_json(self): attributes = {} From 8122a40be064f28dd0aa2ea8567cc2fb2ce4dea8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:48:20 +0100 Subject: [PATCH 3/6] Update moto/dynamodb2/models.py Co-Authored-By: pvbouwel --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index de2a06fd4..62c60efb0 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1127,7 +1127,7 @@ class Table(BaseModel): last_evaluated_key = None size_limit = 1000000 # DynamoDB has a 1MB size limit - item_size = sum([res.size() for res in results]) + item_size = sum(res.size() for res in results) if item_size > size_limit: item_size = idx = 0 while item_size + results[idx].size() < size_limit: From c2b4c397f272f80684c395150159028ed265fd20 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 14 Apr 2020 07:53:15 +0100 Subject: [PATCH 4/6] DDB test - Fix KeySchema, and set BillingMode for easier online testing --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index daae79232..e00a45e1d 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4143,13 +4143,13 @@ def test_dynamodb_max_1mb_limit(): TableName=table_name, KeySchema=[ {"AttributeName": "partition_key", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "SORT"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, ], AttributeDefinitions=[ {"AttributeName": "partition_key", "AttributeType": "S"}, {"AttributeName": "sort_key", "AttributeType": "S"}, ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + BillingMode="PAY_PER_REQUEST", ) # Populate the table @@ -4170,3 +4170,4 @@ def test_dynamodb_max_1mb_limit(): # We shouldn't get everything back - the total result set is well over 1MB assert response["Count"] < len(items) response["LastEvaluatedKey"].shouldnt.be(None) + From a6902e87137da229d63d138b898a63ffd12fe326 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 15 Apr 2020 07:26:09 +0100 Subject: [PATCH 5/6] Update tests/test_dynamodb2/test_dynamodb.py Co-Authored-By: Guilherme Martins Crocetti --- tests/test_dynamodb2/test_dynamodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e00a45e1d..2b4c0969c 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4168,6 +4168,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - assert response["Count"] < len(items) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None) - From 870b34ba7693e88df38d2f2765b972cfda955cee Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 16 Apr 2020 07:09:50 +0100 Subject: [PATCH 6/6] Spacing --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 8a8c69a8c..a0a5e6406 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4213,5 +4213,5 @@ def test_dynamodb_max_1mb_limit(): KeyConditionExpression=Key("partition_key").eq("partition_key_val") ) # We shouldn't get everything back - the total result set is well over 1MB - len(items).should.be.greater_than(response["Count"]) + len(items).should.be.greater_than(response["Count"]) response["LastEvaluatedKey"].shouldnt.be(None)