From cbf03979536a805408b093ee03e90df7942c0b6e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 18 Mar 2020 13:02:07 +0000 Subject: [PATCH] #2255 - CF - Implement FN::Transform and AWS::Include --- moto/cloudformation/parsing.py | 21 ++++++++- moto/cloudwatch/models.py | 10 ++-- moto/logs/models.py | 1 + moto/s3/utils.py | 11 +++++ .../test_cloudformation_stack_integration.py | 47 +++++++++++++++++++ 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index d7e15c7b4..79276c8fc 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals import functools +import json import logging import copy import warnings @@ -24,7 +25,8 @@ from moto.rds import models as rds_models from moto.rds2 import models as rds2_models from moto.redshift import models as redshift_models from moto.route53 import models as route53_models -from moto.s3 import models as s3_models +from moto.s3 import models as s3_models, s3_backend +from moto.s3.utils import bucket_and_name_from_url from moto.sns import models as sns_models from moto.sqs import models as sqs_models from moto.core import ACCOUNT_ID @@ -150,7 +152,10 @@ def clean_json(resource_json, resources_map): map_path = resource_json["Fn::FindInMap"][1:] result = resources_map[map_name] for path in map_path: - result = result[clean_json(path, resources_map)] + if "Fn::Transform" in result: + result = resources_map[clean_json(path, resources_map)] + else: + result = result[clean_json(path, resources_map)] return result if "Fn::GetAtt" in resource_json: @@ -470,6 +475,17 @@ class ResourceMap(collections_abc.Mapping): def load_mapping(self): self._parsed_resources.update(self._template.get("Mappings", {})) + def transform_mapping(self): + for k, v in self._template.get("Mappings", {}).items(): + if "Fn::Transform" in v: + name = v["Fn::Transform"]["Name"] + params = v["Fn::Transform"]["Parameters"] + if name == "AWS::Include": + location = params["Location"] + bucket_name, name = bucket_and_name_from_url(location) + key = s3_backend.get_key(bucket_name, name) + self._parsed_resources.update(json.loads(key.value)) + def load_parameters(self): parameter_slots = self._template.get("Parameters", {}) for parameter_name, parameter in parameter_slots.items(): @@ -515,6 +531,7 @@ class ResourceMap(collections_abc.Mapping): def create(self): self.load_mapping() + self.transform_mapping() self.load_parameters() self.load_conditions() diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index bdba09930..a8a1b1d19 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -5,6 +5,7 @@ from boto3 import Session from moto.core.utils import iso_8601_datetime_without_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError +from moto.logs import logs_backends from datetime import datetime, timedelta from dateutil.tz import tzutc from uuid import uuid4 @@ -428,12 +429,9 @@ class LogGroup(BaseModel): cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - spec = {"LogGroupName": properties["LogGroupName"]} - optional_properties = "Tags".split() - for prop in optional_properties: - if prop in properties: - spec[prop] = properties[prop] - return LogGroup(spec) + log_group_name = properties["LogGroupName"] + tags = properties.get("Tags", {}) + return logs_backends[region_name].create_log_group(log_group_name, tags) cloudwatch_backends = {} diff --git a/moto/logs/models.py b/moto/logs/models.py index 7448319db..5e21d8793 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -405,6 +405,7 @@ class LogsBackend(BaseBackend): if log_group_name in self.groups: raise ResourceAlreadyExistsException() self.groups[log_group_name] = LogGroup(self.region_name, log_group_name, tags) + return self.groups[log_group_name] def ensure_log_group(self, log_group_name, tags): if log_group_name in self.groups: diff --git a/moto/s3/utils.py b/moto/s3/utils.py index 6855c9b25..6ddcfa63e 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -35,6 +35,17 @@ def bucket_name_from_url(url): return None +# 'owi-common-cf', 'snippets/test.json' = bucket_and_name_from_url('s3://owi-common-cf/snippets/test.json') +def bucket_and_name_from_url(url): + prefix = "s3://" + if url.startswith(prefix): + bucket_name = url[len(prefix) : url.index("/", len(prefix))] + key = url[url.index("/", len(prefix)) + 1 :] + return bucket_name, key + else: + return None, None + + REGION_URL_REGEX = re.compile( r"^https?://(s3[-\.](?P.+)\.amazonaws\.com/(.+)|" r"(.+)\.s3[-\.](?P.+)\.amazonaws\.com)/?" diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 5a3181449..a612156c4 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -32,12 +32,14 @@ from moto import ( mock_iam_deprecated, mock_kms, mock_lambda, + mock_logs, mock_rds_deprecated, mock_rds2, mock_rds2_deprecated, mock_redshift, mock_redshift_deprecated, mock_route53_deprecated, + mock_s3, mock_sns_deprecated, mock_sqs, mock_sqs_deprecated, @@ -2332,3 +2334,48 @@ def test_stack_dynamodb_resources_integration(): response["Item"]["Sales"].should.equal(Decimal("10")) response["Item"]["NumberOfSongs"].should.equal(Decimal("5")) response["Item"]["Album"].should.equal("myAlbum") + + +@mock_cloudformation +@mock_logs +@mock_s3 +def test_create_log_group_using_fntransform(): + s3_resource = boto3.resource("s3") + s3_resource.create_bucket( + Bucket="owi-common-cf", + CreateBucketConfiguration={"LocationConstraint": "us-west-2"}, + ) + s3_resource.Object("owi-common-cf", "snippets/test.json").put( + Body=json.dumps({"lgname": {"name": "some-log-group"}}) + ) + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Mappings": { + "EnvironmentMapping": { + "Fn::Transform": { + "Name": "AWS::Include", + "Parameters": {"Location": "s3://owi-common-cf/snippets/test.json"}, + } + } + }, + "Resources": { + "LogGroup": { + "Properties": { + "LogGroupName": { + "Fn::FindInMap": ["EnvironmentMapping", "lgname", "name"] + }, + "RetentionInDays": 90, + }, + "Type": "AWS::Logs::LogGroup", + } + }, + } + + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack( + StackName="test_stack", TemplateBody=json.dumps(template), + ) + + logs_conn = boto3.client("logs", region_name="us-west-2") + log_group = logs_conn.describe_log_groups()["logGroups"][0] + log_group["logGroupName"].should.equal("some-log-group")