From ba1bf09474933976f422179e3e902d3866b0892c Mon Sep 17 00:00:00 2001 From: ImFlog Date: Wed, 5 Feb 2020 09:31:03 +0100 Subject: [PATCH 01/14] Fix UPDATED_NEW return values differences between moto and dynamoDB --- moto/dynamodb2/responses.py | 43 ++++++++++++++++++++++----- tests/test_dynamodb2/test_dynamodb.py | 36 ++++++++++++++++++---- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d3767c3fd..826a9a19c 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -1,9 +1,12 @@ from __future__ import unicode_literals -import itertools + +import copy import json -import six import re +import itertools +import six + from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge @@ -710,7 +713,8 @@ class DynamoHandler(BaseResponse): attribute_updates = self.body.get("AttributeUpdates") expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - existing_item = self.dynamodb_backend.get_item(name, key) + # We need to copy the item in order to avoid it being modified by the update_item operation + existing_item = copy.deepcopy(self.dynamodb_backend.get_item(name, key)) if existing_item: existing_attributes = existing_item.to_json()["Attributes"] else: @@ -796,14 +800,37 @@ class DynamoHandler(BaseResponse): k: v for k, v in existing_attributes.items() if k in changed_attributes } elif return_values == "UPDATED_NEW": - item_dict["Attributes"] = { - k: v - for k, v in item_dict["Attributes"].items() - if k in changed_attributes - } + item_dict["Attributes"] = self._build_updated_new_attributes( + existing_attributes, item_dict["Attributes"] + ) return dynamo_json_dump(item_dict) + def _build_updated_new_attributes(self, original, changed): + if type(changed) != type(original): + return changed + else: + if type(changed) is dict: + return { + key: self._build_updated_new_attributes( + original.get(key, None), changed[key] + ) + for key in changed.keys() + if changed[key] != original.get(key, None) + } + elif type(changed) in (set, list): + if len(changed) != len(original): + return changed + else: + return [ + self._build_updated_new_attributes(original[index], changed[index]) + for index in range(len(changed)) + ] + elif changed != original: + return changed + else: + return None + def describe_limits(self): return json.dumps( { diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 428b58f81..a2ea09c0e 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3412,13 +3412,18 @@ def test_update_supports_list_append(): ) # Update item using list_append expression - client.update_item( + updated_item = client.update_item( TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}}, UpdateExpression="SET crontab = list_append(crontab, :i)", ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, + ReturnValues="UPDATED_NEW", ) + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}} + ) # Verify item is appended to the existing list result = client.get_item( TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}} @@ -3451,15 +3456,19 @@ def test_update_supports_nested_list_append(): ) # Update item using list_append expression - client.update_item( + updated_item = client.update_item( TableName="TestTable", Key={"id": {"S": "nested_list_append"}}, UpdateExpression="SET a.#b = list_append(a.#b, :i)", ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, ExpressionAttributeNames={"#b": "b"}, + ReturnValues="UPDATED_NEW", ) - # Verify item is appended to the existing list + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} + ) result = client.get_item( TableName="TestTable", Key={"id": {"S": "nested_list_append"}} )["Item"] @@ -3491,14 +3500,19 @@ def test_update_supports_multiple_levels_nested_list_append(): ) # Update item using list_append expression - client.update_item( + updated_item = client.update_item( TableName="TestTable", Key={"id": {"S": "nested_list_append"}}, UpdateExpression="SET a.#b.c = list_append(a.#b.#c, :i)", ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, ExpressionAttributeNames={"#b": "b", "#c": "c"}, + ReturnValues="UPDATED_NEW", ) + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}} + ) # Verify item is appended to the existing list result = client.get_item( TableName="TestTable", Key={"id": {"S": "nested_list_append"}} @@ -3532,14 +3546,19 @@ def test_update_supports_nested_list_append_onto_another_list(): ) # Update item using list_append expression - client.update_item( + updated_item = client.update_item( TableName="TestTable", Key={"id": {"S": "list_append_another"}}, UpdateExpression="SET a.#c = list_append(a.#b, :i)", ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, ExpressionAttributeNames={"#b": "b", "#c": "c"}, + ReturnValues="UPDATED_NEW", ) + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} + ) # Verify item is appended to the existing list result = client.get_item( TableName="TestTable", Key={"id": {"S": "list_append_another"}} @@ -3582,13 +3601,18 @@ def test_update_supports_list_append_maps(): ) # Update item using list_append expression - client.update_item( + updated_item = client.update_item( TableName="TestTable", Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}}, UpdateExpression="SET a = list_append(a, :i)", ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}}, + ReturnValues="UPDATED_NEW", ) + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}} + ) # Verify item is appended to the existing list result = client.query( TableName="TestTable", From 3802767817139ca1d287c15cc266c4e114f5ddeb Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 12 Mar 2020 12:25:31 +0000 Subject: [PATCH 02/14] S3 - Add test case to showcase bug when downloading large files --- moto/s3/models.py | 11 +++++- moto/s3/responses.py | 15 ++++++- moto/s3/utils.py | 4 +- tests/test_s3/test_s3.py | 84 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 5a665e27e..67b53b984 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -12,6 +12,7 @@ import codecs import random import string import tempfile +import threading import sys import time import uuid @@ -110,6 +111,7 @@ class FakeKey(BaseModel): self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size) self._max_buffer_size = max_buffer_size self.value = value + self.lock = threading.Lock() @property def version_id(self): @@ -117,8 +119,14 @@ class FakeKey(BaseModel): @property def value(self): + self.lock.acquire() + print("===>value") self._value_buffer.seek(0) - return self._value_buffer.read() + print("===>seek") + r = self._value_buffer.read() + print("===>read") + self.lock.release() + return r @value.setter def value(self, new_value): @@ -1319,6 +1327,7 @@ class S3Backend(BaseBackend): return key def get_key(self, bucket_name, key_name, version_id=None, part_number=None): + print("get_key("+str(bucket_name)+","+str(key_name)+","+str(version_id)+","+str(part_number)+")") key_name = clean_key_name(key_name) bucket = self.get_bucket(bucket_name) key = None diff --git a/moto/s3/responses.py b/moto/s3/responses.py index b74be9a63..15b1d1670 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import re import sys +import threading import six from botocore.awsrequest import AWSPreparedRequest @@ -150,6 +151,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): self.path = "" self.data = {} self.headers = {} + self.lock = threading.Lock() @property def should_autoescape(self): @@ -857,6 +859,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _handle_range_header(self, request, headers, response_content): response_headers = {} length = len(response_content) + print("Length: " + str(length) + " Range: " + str(request.headers.get("range"))) last = length - 1 _, rspec = request.headers.get("range").split("=") if "," in rspec: @@ -874,6 +877,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: return 400, response_headers, "" if begin < 0 or end > last or begin > min(end, last): + print(str(begin)+ " < 0 or " + str(end) + " > " + str(last) + " or " + str(begin) + " > min("+str(end)+","+str(last)+")") return 416, response_headers, "" response_headers["content-range"] = "bytes {0}-{1}/{2}".format( begin, end, length @@ -903,14 +907,20 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_content = response else: status_code, response_headers, response_content = response + print("response received: " + str(len(response_content))) + print(request.headers) if status_code == 200 and "range" in request.headers: - return self._handle_range_header( + self.lock.acquire() + r = self._handle_range_header( request, response_headers, response_content ) + self.lock.release() + return r return status_code, response_headers, response_content def _control_response(self, request, full_url, headers): + print("_control_response") parsed_url = urlparse(full_url) query = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method @@ -1058,12 +1068,14 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): ) def _key_response_get(self, bucket_name, query, key_name, headers): + print("_key_response_get("+str(key_name)+","+str(headers)+")") self._set_action("KEY", "GET", query) self._authenticate_and_authorize_s3_action() response_headers = {} if query.get("uploadId"): upload_id = query["uploadId"][0] + print("UploadID: " + str(upload_id)) parts = self.backend.list_multipart(bucket_name, upload_id) template = self.response_template(S3_MULTIPART_LIST_RESPONSE) return ( @@ -1095,6 +1107,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_headers.update(key.metadata) response_headers.update(key.response_dict) + print("returning 200, " + str(headers) + ", " + str(len(key.value)) + " ( " + str(key_name) + ")") return 200, response_headers, key.value def _key_response_put(self, request, body, bucket_name, query, key_name, headers): diff --git a/moto/s3/utils.py b/moto/s3/utils.py index e22b6b860..50ff1cf34 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -104,7 +104,9 @@ class _VersionedKeyStore(dict): def get(self, key, default=None): try: return self[key] - except (KeyError, IndexError): + except (KeyError, IndexError) as e: + print("Error retrieving " + str(key)) + print(e) pass return default diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 48655ee17..2eef9ef82 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4393,3 +4393,87 @@ def test_s3_config_dict(): assert not logging_bucket["supplementaryConfiguration"].get( "BucketTaggingConfiguration" ) + + +@mock_s3 +def test_delete_downloaded_file(): + # SET UP + filename = '...' + file = open(filename, 'rb') + uploader = PdfFileUploader(file) + boto3.client('s3').create_bucket(Bucket=uploader.bucket_name()) + uploader.upload() + print("================\nUPLOADED\n=================") + # DOWNLOAD + # the following two lines are basically + # boto3.client('s3').download_file(bucket_name, file_name, local_path) + # where bucket_name, file_name and local_path are retrieved from PdfFileUploader + # e.g. boto3.client('s3').download_file("bucket_name", "asdf.pdf", "/tmp/asdf.pdf") + downloader = PdfFileDownloader(uploader.full_bucket_file_name()) + downloader.download() + + downloader.delete_downloaded_file() + + print("Done!") + + +from pathlib import Path +import re +import os +class PdfFileDownloader: + def __init__(self, full_bucket_file_name): + self.bucket_name, self.file_name = self.extract(full_bucket_file_name) + self.s3 = boto3.client('s3') + + def download(self): + try: + self.s3.download_file(self.bucket_name, self.file_name, self.local_path()) + + return self.local_path() + except ClientError as exc: + print("=======") + print(exc) + raise exc + + def local_path(self): + return '/tmp/' + self.file_name.replace('/', '') + + def delete_downloaded_file(self): + if Path(self.local_path()).is_file(): + print("Removing " + str(self.local_path())) + os.remove(self.local_path()) + + def file(self): + return open(self.local_path(), 'rb') + + def extract(self, full_bucket_file_name): + match = re.search(r'([\.a-zA-Z_-]+)\/(.*)', full_bucket_file_name) + + if match and len(match.groups()) == 2: + return (match.groups()[0], match.groups()[1]) + else: + raise RuntimeError(f"Cannot determine bucket and file name for {full_bucket_file_name}") + + +import binascii +class PdfFileUploader: + def __init__(self, file): + self.file = file + date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + random_hex = binascii.b2a_hex(os.urandom(16)).decode('ascii') + self.bucket_file_name = f"{date}_{random_hex}.pdf" + + def upload(self): + self.file.seek(0) + boto3.client('s3').upload_fileobj(self.file, self.bucket_name(), self.bucket_file_name) + + return (self.original_file_name(), self.full_bucket_file_name()) + + def original_file_name(self): + return os.path.basename(self.file.name) + + def bucket_name(self): + return 'test_bucket' #os.environ['AWS_BUCKET_NAME'] + + def full_bucket_file_name(self): + return f"{self.bucket_name()}/{self.bucket_file_name}" From 7f6c6660aa5280ac36c919af8256148df6989c6f Mon Sep 17 00:00:00 2001 From: ImFlog Date: Thu, 12 Mar 2020 17:56:11 +0100 Subject: [PATCH 03/14] Add some new update_new tests --- moto/dynamodb2/responses.py | 4 +++- tests/test_dynamodb2/test_dynamodb.py | 32 +++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 826a9a19c..9e3c3a79b 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -823,7 +823,9 @@ class DynamoHandler(BaseResponse): return changed else: return [ - self._build_updated_new_attributes(original[index], changed[index]) + self._build_updated_new_attributes( + original[index], changed[index] + ) for index in range(len(changed)) ] elif changed != original: diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index a2ea09c0e..05c539721 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -3648,11 +3648,18 @@ def test_update_supports_list_append_with_nested_if_not_exists_operation(): table = dynamo.Table(table_name) table.put_item(Item={"Id": "item-id", "nest1": {"nest2": {}}}) - table.update_item( + updated_item = table.update_item( Key={"Id": "item-id"}, UpdateExpression="SET nest1.nest2.event_history = list_append(if_not_exists(nest1.nest2.event_history, :empty_list), :new_value)", ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, + ReturnValues="UPDATED_NEW", ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"nest1": {"nest2": {"event_history": ["some_value"]}}} + ) + table.get_item(Key={"Id": "item-id"})["Item"].should.equal( {"Id": "item-id", "nest1": {"nest2": {"event_history": ["some_value"]}}} ) @@ -3673,11 +3680,18 @@ def test_update_supports_list_append_with_nested_if_not_exists_operation_and_pro table = dynamo.Table(table_name) table.put_item(Item={"Id": "item-id", "event_history": ["other_value"]}) - table.update_item( + updated_item = table.update_item( Key={"Id": "item-id"}, UpdateExpression="SET event_history = list_append(if_not_exists(event_history, :empty_list), :new_value)", ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, + ReturnValues="UPDATED_NEW", ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"event_history": ["other_value", "some_value"]} + ) + table.get_item(Key={"Id": "item-id"})["Item"].should.equal( {"Id": "item-id", "event_history": ["other_value", "some_value"]} ) @@ -3764,11 +3778,16 @@ def test_update_nested_item_if_original_value_is_none(): ) table = dynamo.Table("origin-rbu-dev") table.put_item(Item={"job_id": "a", "job_details": {"job_name": None}}) - table.update_item( + updated_item = table.update_item( Key={"job_id": "a"}, UpdateExpression="SET job_details.job_name = :output", ExpressionAttributeValues={":output": "updated"}, + ReturnValues="UPDATED_NEW", ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) + table.scan()["Items"][0]["job_details"]["job_name"].should.equal("updated") @@ -3784,11 +3803,16 @@ def test_allow_update_to_item_with_different_type(): table = dynamo.Table("origin-rbu-dev") table.put_item(Item={"job_id": "a", "job_details": {"job_name": {"nested": "yes"}}}) table.put_item(Item={"job_id": "b", "job_details": {"job_name": {"nested": "yes"}}}) - table.update_item( + updated_item = table.update_item( Key={"job_id": "a"}, UpdateExpression="SET job_details.job_name = :output", ExpressionAttributeValues={":output": "updated"}, + ReturnValues="UPDATED_NEW", ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) + table.get_item(Key={"job_id": "a"})["Item"]["job_details"][ "job_name" ].should.be.equal("updated") From d8423b5de0f8770149449b54f7b09ed05419233b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 09:16:12 +0000 Subject: [PATCH 04/14] Optimize content length for large files --- moto/s3/models.py | 14 +++++++------- moto/s3/responses.py | 8 -------- tests/test_s3/test_s3.py | 13 ++----------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 67b53b984..8c2a86f41 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -120,11 +120,9 @@ class FakeKey(BaseModel): @property def value(self): self.lock.acquire() - print("===>value") self._value_buffer.seek(0) - print("===>seek") r = self._value_buffer.read() - print("===>read") + r = copy.copy(r) self.lock.release() return r @@ -138,6 +136,7 @@ class FakeKey(BaseModel): if isinstance(new_value, six.text_type): new_value = new_value.encode(DEFAULT_TEXT_ENCODING) self._value_buffer.write(new_value) + self.contentsize = len(new_value) def copy(self, new_name=None, new_is_versioned=None): r = copy.deepcopy(self) @@ -165,6 +164,7 @@ class FakeKey(BaseModel): self.acl = acl def append_to_value(self, value): + self.contentsize += len(value) self._value_buffer.seek(0, os.SEEK_END) self._value_buffer.write(value) @@ -237,8 +237,7 @@ class FakeKey(BaseModel): @property def size(self): - self._value_buffer.seek(0, os.SEEK_END) - return self._value_buffer.tell() + return self.contentsize @property def storage_class(self): @@ -257,6 +256,7 @@ class FakeKey(BaseModel): state = self.__dict__.copy() state["value"] = self.value del state["_value_buffer"] + del state["lock"] return state def __setstate__(self, state): @@ -266,6 +266,7 @@ class FakeKey(BaseModel): max_size=self._max_buffer_size ) self.value = state["value"] + self.lock = threading.Lock() class FakeMultipart(BaseModel): @@ -292,7 +293,7 @@ class FakeMultipart(BaseModel): etag = etag.replace('"', "") if part is None or part_etag != etag: raise InvalidPart() - if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE: + if last is not None and last.contentsize < UPLOAD_PART_MIN_SIZE: raise EntityTooSmall() md5s.extend(decode_hex(part_etag)[0]) total.extend(part.value) @@ -1327,7 +1328,6 @@ class S3Backend(BaseBackend): return key def get_key(self, bucket_name, key_name, version_id=None, part_number=None): - print("get_key("+str(bucket_name)+","+str(key_name)+","+str(version_id)+","+str(part_number)+")") key_name = clean_key_name(key_name) bucket = self.get_bucket(bucket_name) key = None diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 15b1d1670..4f38e2a9b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -859,7 +859,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): def _handle_range_header(self, request, headers, response_content): response_headers = {} length = len(response_content) - print("Length: " + str(length) + " Range: " + str(request.headers.get("range"))) last = length - 1 _, rspec = request.headers.get("range").split("=") if "," in rspec: @@ -877,7 +876,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): else: return 400, response_headers, "" if begin < 0 or end > last or begin > min(end, last): - print(str(begin)+ " < 0 or " + str(end) + " > " + str(last) + " or " + str(begin) + " > min("+str(end)+","+str(last)+")") return 416, response_headers, "" response_headers["content-range"] = "bytes {0}-{1}/{2}".format( begin, end, length @@ -907,8 +905,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_content = response else: status_code, response_headers, response_content = response - print("response received: " + str(len(response_content))) - print(request.headers) if status_code == 200 and "range" in request.headers: self.lock.acquire() @@ -920,7 +916,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return status_code, response_headers, response_content def _control_response(self, request, full_url, headers): - print("_control_response") parsed_url = urlparse(full_url) query = parse_qs(parsed_url.query, keep_blank_values=True) method = request.method @@ -1068,14 +1063,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): ) def _key_response_get(self, bucket_name, query, key_name, headers): - print("_key_response_get("+str(key_name)+","+str(headers)+")") self._set_action("KEY", "GET", query) self._authenticate_and_authorize_s3_action() response_headers = {} if query.get("uploadId"): upload_id = query["uploadId"][0] - print("UploadID: " + str(upload_id)) parts = self.backend.list_multipart(bucket_name, upload_id) template = self.response_template(S3_MULTIPART_LIST_RESPONSE) return ( @@ -1107,7 +1100,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): response_headers.update(key.metadata) response_headers.update(key.response_dict) - print("returning 200, " + str(headers) + ", " + str(len(key.value)) + " ( " + str(key_name) + ")") return 200, response_headers, key.value def _key_response_put(self, request, body, bucket_name, query, key_name, headers): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 2eef9ef82..7b9f2c726 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4398,24 +4398,17 @@ def test_s3_config_dict(): @mock_s3 def test_delete_downloaded_file(): # SET UP - filename = '...' + filename = 'some_large_file.pdf' file = open(filename, 'rb') uploader = PdfFileUploader(file) boto3.client('s3').create_bucket(Bucket=uploader.bucket_name()) uploader.upload() - print("================\nUPLOADED\n=================") - # DOWNLOAD - # the following two lines are basically - # boto3.client('s3').download_file(bucket_name, file_name, local_path) - # where bucket_name, file_name and local_path are retrieved from PdfFileUploader - # e.g. boto3.client('s3').download_file("bucket_name", "asdf.pdf", "/tmp/asdf.pdf") + downloader = PdfFileDownloader(uploader.full_bucket_file_name()) downloader.download() downloader.delete_downloaded_file() - print("Done!") - from pathlib import Path import re @@ -4431,8 +4424,6 @@ class PdfFileDownloader: return self.local_path() except ClientError as exc: - print("=======") - print(exc) raise exc def local_path(self): From e2434cbf6f4f939c99fd4d81cbad285702251fd1 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 09:18:38 +0000 Subject: [PATCH 05/14] Remove unnecessary lock --- moto/s3/responses.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 4f38e2a9b..b74be9a63 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import re import sys -import threading import six from botocore.awsrequest import AWSPreparedRequest @@ -151,7 +150,6 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): self.path = "" self.data = {} self.headers = {} - self.lock = threading.Lock() @property def should_autoescape(self): @@ -907,12 +905,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): status_code, response_headers, response_content = response if status_code == 200 and "range" in request.headers: - self.lock.acquire() - r = self._handle_range_header( + return self._handle_range_header( request, response_headers, response_content ) - self.lock.release() - return r return status_code, response_headers, response_content def _control_response(self, request, full_url, headers): From 5e4736e23392079c20bb283a5ceb2c8e8d6bacf4 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 09:19:57 +0000 Subject: [PATCH 06/14] Remove unnecessary print-statements --- moto/s3/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/moto/s3/utils.py b/moto/s3/utils.py index 50ff1cf34..e22b6b860 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -104,9 +104,7 @@ class _VersionedKeyStore(dict): def get(self, key, default=None): try: return self[key] - except (KeyError, IndexError) as e: - print("Error retrieving " + str(key)) - print(e) + except (KeyError, IndexError): pass return default From 410d9ee90186d5e83c81f97dc93b6a24faf62b39 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 09:21:33 +0000 Subject: [PATCH 07/14] Remove test that only runs locally --- tests/test_s3/test_s3.py | 75 ---------------------------------------- 1 file changed, 75 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 7b9f2c726..48655ee17 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4393,78 +4393,3 @@ def test_s3_config_dict(): assert not logging_bucket["supplementaryConfiguration"].get( "BucketTaggingConfiguration" ) - - -@mock_s3 -def test_delete_downloaded_file(): - # SET UP - filename = 'some_large_file.pdf' - file = open(filename, 'rb') - uploader = PdfFileUploader(file) - boto3.client('s3').create_bucket(Bucket=uploader.bucket_name()) - uploader.upload() - - downloader = PdfFileDownloader(uploader.full_bucket_file_name()) - downloader.download() - - downloader.delete_downloaded_file() - - -from pathlib import Path -import re -import os -class PdfFileDownloader: - def __init__(self, full_bucket_file_name): - self.bucket_name, self.file_name = self.extract(full_bucket_file_name) - self.s3 = boto3.client('s3') - - def download(self): - try: - self.s3.download_file(self.bucket_name, self.file_name, self.local_path()) - - return self.local_path() - except ClientError as exc: - raise exc - - def local_path(self): - return '/tmp/' + self.file_name.replace('/', '') - - def delete_downloaded_file(self): - if Path(self.local_path()).is_file(): - print("Removing " + str(self.local_path())) - os.remove(self.local_path()) - - def file(self): - return open(self.local_path(), 'rb') - - def extract(self, full_bucket_file_name): - match = re.search(r'([\.a-zA-Z_-]+)\/(.*)', full_bucket_file_name) - - if match and len(match.groups()) == 2: - return (match.groups()[0], match.groups()[1]) - else: - raise RuntimeError(f"Cannot determine bucket and file name for {full_bucket_file_name}") - - -import binascii -class PdfFileUploader: - def __init__(self, file): - self.file = file - date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') - random_hex = binascii.b2a_hex(os.urandom(16)).decode('ascii') - self.bucket_file_name = f"{date}_{random_hex}.pdf" - - def upload(self): - self.file.seek(0) - boto3.client('s3').upload_fileobj(self.file, self.bucket_name(), self.bucket_file_name) - - return (self.original_file_name(), self.full_bucket_file_name()) - - def original_file_name(self): - return os.path.basename(self.file.name) - - def bucket_name(self): - return 'test_bucket' #os.environ['AWS_BUCKET_NAME'] - - def full_bucket_file_name(self): - return f"{self.bucket_name()}/{self.bucket_file_name}" From b7da6b948152f96884c2e81bc06876a8e8e60713 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 15:41:50 +0000 Subject: [PATCH 08/14] #2813 - DynamoDB - Add Global Index Status --- moto/dynamodb2/models.py | 4 ++++ tests/test_dynamodb2/test_dynamodb_table_with_range_key.py | 1 + 2 files changed, 5 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 1527821ed..91980ab0d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -679,6 +679,10 @@ class Table(BaseModel): self.throughput["NumberOfDecreasesToday"] = 0 self.indexes = indexes self.global_indexes = global_indexes if global_indexes else [] + for index in self.global_indexes: + index[ + "IndexStatus" + ] = "ACTIVE" # One of 'CREATING'|'UPDATING'|'DELETING'|'ACTIVE' self.created_at = datetime.datetime.utcnow() self.items = defaultdict(dict) self.table_arn = self._generate_arn(table_name) diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index 7c7770874..c433a3a31 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -574,6 +574,7 @@ def test_create_with_global_indexes(): "ReadCapacityUnits": 6, "WriteCapacityUnits": 1, }, + "IndexStatus": "ACTIVE", } ] ) From 3fab3f572f3da3470be1032775a3cf77dd7582f7 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 16:09:42 +0000 Subject: [PATCH 09/14] #2773 - CloudFormation - Set CreationDate --- moto/cloudformation/models.py | 6 ++++++ moto/cloudformation/responses.py | 4 ++-- .../test_cloudformation_stack_crud_boto3.py | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index b32d63b32..8136e353d 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -8,6 +8,7 @@ from boto3 import Session from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel +from moto.core.utils import iso_8601_datetime_without_milliseconds from .parsing import ResourceMap, OutputMap from .utils import ( @@ -240,6 +241,7 @@ class FakeStack(BaseModel): self.output_map = self._create_output_map() self._add_stack_event("CREATE_COMPLETE") self.status = "CREATE_COMPLETE" + self.creation_time = datetime.utcnow() def _create_resource_map(self): resource_map = ResourceMap( @@ -259,6 +261,10 @@ class FakeStack(BaseModel): output_map.create() return output_map + @property + def creation_time_iso_8601(self): + return iso_8601_datetime_without_milliseconds(self.creation_time) + def _add_stack_event( self, resource_status, resource_status_reason=None, resource_properties=None ): diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 77a3051fd..782d68946 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -662,7 +662,7 @@ DESCRIBE_STACKS_TEMPLATE = """ {{ stack.name }} {{ stack.stack_id }} - 2010-07-27T22:28:28Z + {{ stack.creation_time_iso_8601 }} {{ stack.status }} {% if stack.notification_arns %} @@ -803,7 +803,7 @@ LIST_STACKS_RESPONSE = """ {{ stack.stack_id }} {{ stack.status }} {{ stack.name }} - 2011-05-23T15:47:44Z + {{ stack.creation_time_iso_8601 }} {{ stack.description }} {% endfor %} diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index b7e86a1d5..5444c2278 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals import json from collections import OrderedDict +from datetime import datetime, timedelta +import pytz import boto3 from botocore.exceptions import ClientError @@ -911,6 +913,10 @@ def test_describe_stack_by_name(): stack = cf_conn.describe_stacks(StackName="test_stack")["Stacks"][0] stack["StackName"].should.equal("test_stack") + two_secs_ago = datetime.now(tz=pytz.UTC) - timedelta(seconds=2) + assert ( + two_secs_ago < stack["CreationTime"] < datetime.now(tz=pytz.UTC) + ), "Stack should have been created recently" @mock_cloudformation From 67c7fce85ecf95fdfbc8d768b7839cdb9ff00d5f Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 16:28:49 +0000 Subject: [PATCH 10/14] #2760 - DynamoDB - Ensure proper ordering for Numeric sort keys --- moto/dynamodb2/models.py | 7 +++- tests/test_dynamodb2/test_dynamodb.py | 58 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 1527821ed..a80b3211d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -981,8 +981,13 @@ class Table(BaseModel): if index_name: if index_range_key: + + # Convert to float if necessary to ensure proper ordering + def conv(x): + return float(x.value) if x.type == "N" else x.value + results.sort( - key=lambda item: item.attrs[index_range_key["AttributeName"]].value + key=lambda item: conv(item.attrs[index_range_key["AttributeName"]]) if item.attrs.get(index_range_key["AttributeName"]) else None ) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 82f82ccc9..2b9475b9e 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4026,3 +4026,61 @@ def test_valid_transact_get_items(): "Table": {"CapacityUnits": 2.0, "ReadCapacityUnits": 2.0,}, } ) + + +@mock_dynamodb2 +def test_gsi_verify_negative_number_order(): + table_schema = { + "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], + "GlobalSecondaryIndexes": [ + { + "IndexName": "GSI-K1", + "KeySchema": [ + {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, + {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY",}, + } + ], + "AttributeDefinitions": [ + {"AttributeName": "partitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1SortKey", "AttributeType": "N"}, + ], + } + + item1 = { + "partitionKey": "pk-1", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("-0.6"), + } + + item2 = { + "partitionKey": "pk-2", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("-0.7"), + } + + item3 = { + "partitionKey": "pk-3", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("0.7"), + } + + dynamodb = boto3.resource("dynamodb") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + table = dynamodb.Table("test-table") + table.put_item(Item=item3) + table.put_item(Item=item1) + table.put_item(Item=item2) + + resp = table.query( + KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-k1"), + IndexName="GSI-K1", + ) + # Items should be ordered with the lowest number first + [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( + [-0.7, -0.6, 0.7] + ) From aead80c392942d95a6437689d321c564739b795f Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 17 Mar 2020 17:11:35 +0000 Subject: [PATCH 11/14] Add missing region --- tests/test_dynamodb2/test_dynamodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 2b9475b9e..5d39e3805 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -4067,7 +4067,7 @@ def test_gsi_verify_negative_number_order(): "gsiK1SortKey": Decimal("0.7"), } - dynamodb = boto3.resource("dynamodb") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema ) From a9cb5b566dc6615f41b18b77fc4f5f3071c04e03 Mon Sep 17 00:00:00 2001 From: ImFlog Date: Tue, 17 Mar 2020 18:35:38 +0100 Subject: [PATCH 12/14] Python 2.X, fix missing neq in DynamoType --- moto/dynamodb2/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 8e5a61755..65dd2c3f7 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -146,6 +146,9 @@ class DynamoType(object): def __eq__(self, other): return self.type == other.type and self.value == other.value + def __ne__(self, other): + return self.type != other.type or self.value != other.value + def __lt__(self, other): return self.cast_value < other.cast_value From f0cab68208ae023c9e47976efdd6c9edd1cc3bf6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 18 Mar 2020 11:46:44 +0000 Subject: [PATCH 13/14] #2264 - SES - Ensure verify_email_address works with display names --- moto/ses/models.py | 2 ++ tests/test_ses/test_ses_boto3.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/moto/ses/models.py b/moto/ses/models.py index 4b6ce52c8..91241f706 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -91,9 +91,11 @@ class SESBackend(BaseBackend): return host in self.domains def verify_email_identity(self, address): + _, address = parseaddr(address) self.addresses.append(address) def verify_email_address(self, address): + _, address = parseaddr(address) self.email_addresses.append(address) def verify_domain(self, domain): diff --git a/tests/test_ses/test_ses_boto3.py b/tests/test_ses/test_ses_boto3.py index ee7c92aa1..de8aa0813 100644 --- a/tests/test_ses/test_ses_boto3.py +++ b/tests/test_ses/test_ses_boto3.py @@ -214,3 +214,16 @@ def test_send_raw_email_without_source_or_from(): kwargs = dict(RawMessage={"Data": message.as_string()}) conn.send_raw_email.when.called_with(**kwargs).should.throw(ClientError) + + +@mock_ses +def test_send_email_notification_with_encoded_sender(): + sender = "Foo " + conn = boto3.client("ses", region_name="us-east-1") + conn.verify_email_identity(EmailAddress=sender) + response = conn.send_email( + Source=sender, + Destination={"ToAddresses": ["your.friend@hotmail.com"]}, + Message={"Subject": {"Data": "hi",}, "Body": {"Text": {"Data": "there",}}}, + ) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) From cbf03979536a805408b093ee03e90df7942c0b6e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 18 Mar 2020 13:02:07 +0000 Subject: [PATCH 14/14] #2255 - CF - Implement FN::Transform and AWS::Include --- moto/cloudformation/parsing.py | 21 ++++++++- moto/cloudwatch/models.py | 10 ++-- moto/logs/models.py | 1 + moto/s3/utils.py | 11 +++++ .../test_cloudformation_stack_integration.py | 47 +++++++++++++++++++ 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index d7e15c7b4..79276c8fc 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals import functools +import json import logging import copy import warnings @@ -24,7 +25,8 @@ from moto.rds import models as rds_models from moto.rds2 import models as rds2_models from moto.redshift import models as redshift_models from moto.route53 import models as route53_models -from moto.s3 import models as s3_models +from moto.s3 import models as s3_models, s3_backend +from moto.s3.utils import bucket_and_name_from_url from moto.sns import models as sns_models from moto.sqs import models as sqs_models from moto.core import ACCOUNT_ID @@ -150,7 +152,10 @@ def clean_json(resource_json, resources_map): map_path = resource_json["Fn::FindInMap"][1:] result = resources_map[map_name] for path in map_path: - result = result[clean_json(path, resources_map)] + if "Fn::Transform" in result: + result = resources_map[clean_json(path, resources_map)] + else: + result = result[clean_json(path, resources_map)] return result if "Fn::GetAtt" in resource_json: @@ -470,6 +475,17 @@ class ResourceMap(collections_abc.Mapping): def load_mapping(self): self._parsed_resources.update(self._template.get("Mappings", {})) + def transform_mapping(self): + for k, v in self._template.get("Mappings", {}).items(): + if "Fn::Transform" in v: + name = v["Fn::Transform"]["Name"] + params = v["Fn::Transform"]["Parameters"] + if name == "AWS::Include": + location = params["Location"] + bucket_name, name = bucket_and_name_from_url(location) + key = s3_backend.get_key(bucket_name, name) + self._parsed_resources.update(json.loads(key.value)) + def load_parameters(self): parameter_slots = self._template.get("Parameters", {}) for parameter_name, parameter in parameter_slots.items(): @@ -515,6 +531,7 @@ class ResourceMap(collections_abc.Mapping): def create(self): self.load_mapping() + self.transform_mapping() self.load_parameters() self.load_conditions() diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index bdba09930..a8a1b1d19 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -5,6 +5,7 @@ from boto3 import Session from moto.core.utils import iso_8601_datetime_without_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError +from moto.logs import logs_backends from datetime import datetime, timedelta from dateutil.tz import tzutc from uuid import uuid4 @@ -428,12 +429,9 @@ class LogGroup(BaseModel): cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - spec = {"LogGroupName": properties["LogGroupName"]} - optional_properties = "Tags".split() - for prop in optional_properties: - if prop in properties: - spec[prop] = properties[prop] - return LogGroup(spec) + log_group_name = properties["LogGroupName"] + tags = properties.get("Tags", {}) + return logs_backends[region_name].create_log_group(log_group_name, tags) cloudwatch_backends = {} diff --git a/moto/logs/models.py b/moto/logs/models.py index 7448319db..5e21d8793 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -405,6 +405,7 @@ class LogsBackend(BaseBackend): if log_group_name in self.groups: raise ResourceAlreadyExistsException() self.groups[log_group_name] = LogGroup(self.region_name, log_group_name, tags) + return self.groups[log_group_name] def ensure_log_group(self, log_group_name, tags): if log_group_name in self.groups: diff --git a/moto/s3/utils.py b/moto/s3/utils.py index 6855c9b25..6ddcfa63e 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -35,6 +35,17 @@ def bucket_name_from_url(url): return None +# 'owi-common-cf', 'snippets/test.json' = bucket_and_name_from_url('s3://owi-common-cf/snippets/test.json') +def bucket_and_name_from_url(url): + prefix = "s3://" + if url.startswith(prefix): + bucket_name = url[len(prefix) : url.index("/", len(prefix))] + key = url[url.index("/", len(prefix)) + 1 :] + return bucket_name, key + else: + return None, None + + REGION_URL_REGEX = re.compile( r"^https?://(s3[-\.](?P.+)\.amazonaws\.com/(.+)|" r"(.+)\.s3[-\.](?P.+)\.amazonaws\.com)/?" diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 5a3181449..a612156c4 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -32,12 +32,14 @@ from moto import ( mock_iam_deprecated, mock_kms, mock_lambda, + mock_logs, mock_rds_deprecated, mock_rds2, mock_rds2_deprecated, mock_redshift, mock_redshift_deprecated, mock_route53_deprecated, + mock_s3, mock_sns_deprecated, mock_sqs, mock_sqs_deprecated, @@ -2332,3 +2334,48 @@ def test_stack_dynamodb_resources_integration(): response["Item"]["Sales"].should.equal(Decimal("10")) response["Item"]["NumberOfSongs"].should.equal(Decimal("5")) response["Item"]["Album"].should.equal("myAlbum") + + +@mock_cloudformation +@mock_logs +@mock_s3 +def test_create_log_group_using_fntransform(): + s3_resource = boto3.resource("s3") + s3_resource.create_bucket( + Bucket="owi-common-cf", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + s3_resource.Object("owi-common-cf", "snippets/test.json").put( + Body=json.dumps({"lgname": {"name": "some-log-group"}}) + ) + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Mappings": { + "EnvironmentMapping": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "s3://owi-common-cf/snippets/test.json"}, + } + } + }, + "Resources": { + "LogGroup": { + "Properties": { + "LogGroupName": { + "Fn::FindInMap": ["EnvironmentMapping", "lgname", "name"] + }, + "RetentionInDays": 90, + }, + "Type": "AWS::Logs::LogGroup", + } + }, + } + + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(template), + ) + + logs_conn = boto3.client("logs", region_name="us-west-2") + log_group = logs_conn.describe_log_groups()["logGroups"][0] + log_group["logGroupName"].should.equal("some-log-group")