From a1549b04b43be71c07b3b391ac5b391b40163d8c Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 8 Jun 2017 11:38:29 -0400 Subject: [PATCH 1/5] Add Fn::Split and Fn::Select support --- moto/cloudformation/parsing.py | 9 ++++++ .../test_cloudformation/test_stack_parsing.py | 28 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index eee6aa8e7..09b4530af 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -162,6 +162,15 @@ def clean_json(resource_json, resources_map): if cleaned_val else '{0}'.format(val)) return resource_json['Fn::Join'][0].join(join_list) + if 'Fn::Split' in resource_json: + to_split = clean_json(resource_json['Fn::Split'][1], resources_map) + return to_split.split(resource_json['Fn::Split'][0]) + + if 'Fn::Select' in resource_json: + select_index = int(resource_json['Fn::Select'][0]) + select_list = clean_json(resource_json['Fn::Select'][1], resources_map) + return select_list[select_index] + cleaned_json = {} for key, value in resource_json.items(): cleaned_val = clean_json(value, resources_map) diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index 610b02325..7b582b9b5 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -72,6 +72,19 @@ get_attribute_output = { } } +split_select_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Queue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::Select": [ "1", {"Fn::Split": [ "-", "123-myqueue" ] } ] }, + "VisibilityTimeout": 60, + } + } + } +} + outputs_template = dict(list(dummy_template.items()) + list(output_dict.items())) bad_outputs_template = dict( @@ -85,6 +98,7 @@ output_type_template_json = json.dumps(outputs_template) bad_output_template_json = json.dumps(bad_outputs_template) get_attribute_outputs_template_json = json.dumps( get_attribute_outputs_template) +split_select_template_json = json.dumps(split_select_template) def test_parse_stack_resources(): @@ -266,3 +280,17 @@ def test_reference_other_conditions(): resources_map={}, condition_map={"OtherCondition": True}, ).should.equal(False) + + +def test_parse_split_and_select(): + stack = FakeStack( + stack_id="test_id", + name="test_stack", + template=split_select_template_json, + parameters={}, + region_name='us-west-1') + + stack.resource_map.should.have.length_of(1) + queue = stack.resource_map['Queue'] + queue.name.should.equal("myqueue") + From 711dbaf4fdc34017d5147988bece77ac8b2e68ec Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 8 Jun 2017 13:30:17 -0400 Subject: [PATCH 2/5] Simplify Fn::Join parsing --- moto/cloudformation/parsing.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 09b4530af..71a60371a 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -155,12 +155,8 @@ def clean_json(resource_json, resources_map): return clean_json(false_value, resources_map) if 'Fn::Join' in resource_json: - join_list = [] - for val in resource_json['Fn::Join'][1]: - cleaned_val = clean_json(val, resources_map) - join_list.append('{0}'.format(cleaned_val) - if cleaned_val else '{0}'.format(val)) - return resource_json['Fn::Join'][0].join(join_list) + join_list = clean_json(resource_json['Fn::Join'][1], resources_map) + return resource_json['Fn::Join'][0].join([str(x) for x in join_list]) if 'Fn::Split' in resource_json: to_split = clean_json(resource_json['Fn::Split'][1], resources_map) From d3faaad46b655ce50048d0d0819d618886804a43 Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 8 Jun 2017 15:21:32 -0400 Subject: [PATCH 3/5] Add Fn::Sub support --- moto/cloudformation/parsing.py | 20 +++++++++++ .../test_cloudformation/test_stack_parsing.py | 33 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 71a60371a..744b1d08e 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -4,6 +4,7 @@ import functools import logging import copy import warnings +import re from moto.autoscaling import models as autoscaling_models from moto.awslambda import models as lambda_models @@ -167,6 +168,25 @@ def clean_json(resource_json, resources_map): select_list = clean_json(resource_json['Fn::Select'][1], resources_map) return select_list[select_index] + if 'Fn::Sub' in resource_json: + if isinstance(resource_json['Fn::Sub'], list): + warnings.warn( + "Tried to parse Fn::Sub with variable mapping but it's not supported by moto's CloudFormation implementation") + else: + fn_sub_value = clean_json(resource_json['Fn::Sub'], resources_map) + to_sub = re.findall('(?=\${)[^!^"]*?}', fn_sub_value) + literals = re.findall('(?=\${!)[^"]*?}', fn_sub_value) + for sub in to_sub: + if '.' in sub: + cleaned_ref = clean_json({'Fn::GetAtt': re.findall('(?<=\${)[^"]*?(?=})', sub)[0].split('.')}, resources_map) + else: + cleaned_ref = clean_json({'Ref': re.findall('(?<=\${)[^"]*?(?=})', sub)[0]}, resources_map) + fn_sub_value = fn_sub_value.replace(sub, cleaned_ref) + for literal in literals: + fn_sub_value = fn_sub_value.replace(literal, literal.replace('!', '')) + return fn_sub_value + pass + cleaned_json = {} for key, value in resource_json.items(): cleaned_val = clean_json(value, resources_map) diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index 7b582b9b5..594515468 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -85,6 +85,26 @@ split_select_template = { } } +sub_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Queue1": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::Sub": '${AWS::StackName}-queue-${!Literal}'}, + "VisibilityTimeout": 60, + } + }, + "Queue2": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::Sub": '${Queue1.QueueName}'}, + "VisibilityTimeout": 60, + } + }, + } +} + outputs_template = dict(list(dummy_template.items()) + list(output_dict.items())) bad_outputs_template = dict( @@ -99,6 +119,7 @@ bad_output_template_json = json.dumps(bad_outputs_template) get_attribute_outputs_template_json = json.dumps( get_attribute_outputs_template) split_select_template_json = json.dumps(split_select_template) +sub_template_json = json.dumps(sub_template) def test_parse_stack_resources(): @@ -294,3 +315,15 @@ def test_parse_split_and_select(): queue = stack.resource_map['Queue'] queue.name.should.equal("myqueue") + +def test_sub(): + stack = FakeStack( + stack_id="test_id", + name="test_stack", + template=sub_template_json, + parameters={}, + region_name='us-west-1') + + queue1 = stack.resource_map['Queue1'] + queue2 = stack.resource_map['Queue2'] + queue2.name.should.equal(queue1.name) From 8e4c79625c4008ee5382d506ad80248ee647f4df Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 8 Jun 2017 15:33:14 -0400 Subject: [PATCH 4/5] Clean Export name and value before appending to exports --- moto/cloudformation/parsing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 744b1d08e..8877b90c7 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -522,7 +522,9 @@ class OutputMap(collections.Mapping): if self.outputs: for key, value in self._output_json_map.items(): if value.get('Export'): - exports.append(Export(self._stack_id, value['Export'].get('Name'), value.get('Value'))) + cleaned_name = clean_json(value['Export'].get('Name'), self._resource_map) + cleaned_value = clean_json(value.get('Value'), self._resource_map) + exports.append(Export(self._stack_id, cleaned_name, cleaned_value)) return exports def create(self): From f5106f2cc811efdb4464e1de6ff4dacae7427839 Mon Sep 17 00:00:00 2001 From: Jessie Nadler Date: Thu, 8 Jun 2017 15:33:28 -0400 Subject: [PATCH 5/5] Add Fn::ImportValue support --- moto/cloudformation/models.py | 6 +- moto/cloudformation/parsing.py | 9 ++- .../test_cloudformation_stack_crud_boto3.py | 36 +++++++++++- .../test_cloudformation/test_stack_parsing.py | 55 ++++++++++++++++++- 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index c25103a4c..ec922d8f5 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -15,7 +15,7 @@ from .exceptions import ValidationError class FakeStack(BaseModel): - def __init__(self, stack_id, name, template, parameters, region_name, notification_arns=None, tags=None, role_arn=None): + def __init__(self, stack_id, name, template, parameters, region_name, notification_arns=None, tags=None, role_arn=None, cross_stack_resources=None): self.stack_id = stack_id self.name = name self.template = template @@ -30,6 +30,7 @@ class FakeStack(BaseModel): resource_status_reason="User Initiated") self.description = self.template_dict.get('Description') + self.cross_stack_resources = cross_stack_resources or [] self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() self._add_stack_event("CREATE_COMPLETE") @@ -37,7 +38,7 @@ class FakeStack(BaseModel): def _create_resource_map(self): resource_map = ResourceMap( - self.stack_id, self.name, self.parameters, self.tags, self.region_name, self.template_dict) + self.stack_id, self.name, self.parameters, self.tags, self.region_name, self.template_dict, self.cross_stack_resources) resource_map.create() return resource_map @@ -148,6 +149,7 @@ class CloudFormationBackend(BaseBackend): notification_arns=notification_arns, tags=tags, role_arn=role_arn, + cross_stack_resources=self.exports, ) self.stacks[stack_id] = new_stack self._validate_export_uniqueness(new_stack) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 8877b90c7..928cd68e0 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -187,6 +187,12 @@ def clean_json(resource_json, resources_map): return fn_sub_value pass + if 'Fn::ImportValue' in resource_json: + cleaned_val = clean_json(resource_json['Fn::ImportValue'], resources_map) + values = [x.value for x in resources_map.cross_stack_resources.values() if x.name == cleaned_val] + if any(values): + return values[0] + cleaned_json = {} for key, value in resource_json.items(): cleaned_val = clean_json(value, resources_map) @@ -326,13 +332,14 @@ class ResourceMap(collections.Mapping): each resources is passed this lazy map that it can grab dependencies from. """ - def __init__(self, stack_id, stack_name, parameters, tags, region_name, template): + def __init__(self, stack_id, stack_name, parameters, tags, region_name, template, cross_stack_resources): self._template = template self._resource_json_map = template['Resources'] self._region_name = region_name self.input_parameters = parameters self.tags = copy.deepcopy(tags) self.resolved_parameters = {} + self.cross_stack_resources = cross_stack_resources # Create the default resources self._parsed_resources = { diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index ba324985f..e428d1f63 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -5,7 +5,7 @@ import boto import boto.s3 import boto.s3.key from botocore.exceptions import ClientError -from moto import mock_cloudformation, mock_s3 +from moto import mock_cloudformation, mock_s3, mock_sqs import json import sure # noqa @@ -80,9 +80,23 @@ dummy_output_template = { } } +dummy_import_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Queue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::ImportValue": 'My VPC ID'}, + "VisibilityTimeout": 60, + } + } + } +} + dummy_template_json = json.dumps(dummy_template) dummy_update_template_json = json.dumps(dummy_template) dummy_output_template_json = json.dumps(dummy_output_template) +dummy_import_template_json = json.dumps(dummy_import_template) @mock_cloudformation @@ -501,3 +515,23 @@ def test_export_names_must_be_unique(): StackName="test_stack", TemplateBody=dummy_output_template_json, ) + +@mock_sqs +@mock_cloudformation +def test_stack_with_imports(): + cf = boto3.resource('cloudformation', region_name='us-east-1') + ec2_resource = boto3.resource('sqs', region_name='us-east-1') + + output_stack = cf.create_stack( + StackName="test_stack1", + TemplateBody=dummy_output_template_json, + ) + import_stack = cf.create_stack( + StackName="test_stack2", + TemplateBody=dummy_import_template_json + ) + + output_stack.outputs.should.have.length_of(1) + output = output_stack.outputs[0]['OutputValue'] + queue = ec2_resource.get_queue_by_name(QueueName=output) + queue.should_not.be.none diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index 594515468..ee53e9a68 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -7,7 +7,7 @@ import sure # noqa from moto.cloudformation.exceptions import ValidationError from moto.cloudformation.models import FakeStack -from moto.cloudformation.parsing import resource_class_from_type, parse_condition +from moto.cloudformation.parsing import resource_class_from_type, parse_condition, Export from moto.sqs.models import Queue from moto.s3.models import FakeBucket from boto.cloudformation.stack import Output @@ -105,6 +105,38 @@ sub_template = { } } +export_value_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Queue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::Sub": '${AWS::StackName}-queue'}, + "VisibilityTimeout": 60, + } + } + }, + "Outputs": { + "Output1": { + "Value": "value", + "Export": {"Name": 'queue-us-west-1'} + } + } +} + +import_value_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Queue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": {"Fn::ImportValue": 'queue-us-west-1'}, + "VisibilityTimeout": 60, + } + } + } +} + outputs_template = dict(list(dummy_template.items()) + list(output_dict.items())) bad_outputs_template = dict( @@ -120,6 +152,8 @@ get_attribute_outputs_template_json = json.dumps( get_attribute_outputs_template) split_select_template_json = json.dumps(split_select_template) sub_template_json = json.dumps(sub_template) +export_value_template_json = json.dumps(export_value_template) +import_value_template_json = json.dumps(import_value_template) def test_parse_stack_resources(): @@ -327,3 +361,22 @@ def test_sub(): queue1 = stack.resource_map['Queue1'] queue2 = stack.resource_map['Queue2'] queue2.name.should.equal(queue1.name) + + +def test_import(): + export_stack = FakeStack( + stack_id="test_id", + name="test_stack", + template=export_value_template_json, + parameters={}, + region_name='us-west-1') + import_stack = FakeStack( + stack_id="test_id", + name="test_stack", + template=import_value_template_json, + parameters={}, + region_name='us-west-1', + cross_stack_resources={export_stack.exports[0].value: export_stack.exports[0]}) + + queue = import_stack.resource_map['Queue'] + queue.name.should.equal("value")