Merge pull request #964 from whummer/feat/cloudformation-models

Add extended CloudFormation models for Lambda and DynamoDB

moto/awslambda/models.py

@@ -4,6 +4,7 @@ import base64
 import datetime
 import hashlib
 import io
+import os
 import json
 import sys
 import zipfile
@@ -16,12 +17,12 @@ except:
 import boto.awslambda
 from moto.core import BaseBackend, BaseModel
 from moto.s3.models import s3_backend
-from moto.s3.exceptions import MissingBucket
+from moto.s3.exceptions import MissingBucket, MissingKey
 
 
 class LambdaFunction(BaseModel):
 
-    def __init__(self, spec):
+    def __init__(self, spec, validate_s3=True):
         # required
         self.code = spec['Code']
         self.function_name = spec['FunctionName']
@@ -58,24 +59,25 @@ class LambdaFunction(BaseModel):
             self.code_size = len(to_unzip_code)
             self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest()
         else:
-            # validate s3 bucket
+            # validate s3 bucket and key
+            key = None
             try:
                 # FIXME: does not validate bucket region
                 key = s3_backend.get_key(
                     self.code['S3Bucket'], self.code['S3Key'])
             except MissingBucket:
-                raise ValueError(
+                if do_validate_s3():
-                    "InvalidParameterValueException",
+                    raise ValueError(
-                    "Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist")
+                        "InvalidParameterValueException",
-            else:
+                        "Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist")
-                # validate s3 key
+            except MissingKey:
-                if key is None:
+                if do_validate_s3():
                     raise ValueError(
                         "InvalidParameterValueException",
                         "Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.")
-                else:
+            if key:
-                    self.code_size = key.size
+                self.code_size = key.size
-                    self.code_sha_256 = hashlib.sha256(key.value).hexdigest()
+                self.code_sha_256 = hashlib.sha256(key.value).hexdigest()
         self.function_arn = 'arn:aws:lambda:123456789012:function:{0}'.format(
             self.function_name)
 
@@ -209,6 +211,13 @@ class LambdaFunction(BaseModel):
         fn = backend.create_function(spec)
         return fn
 
+    def get_cfn_attribute(self, attribute_name):
+        from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
+        if attribute_name == 'Arn':
+            region = 'us-east-1'
+            return 'arn:aws:lambda:{0}:123456789012:function:{1}'.format(region, self.function_name)
+        raise UnformattedGetAttTemplateException()
+
     @staticmethod
     def _create_zipfile_from_plaintext_code(code):
         zip_output = io.BytesIO()
@@ -219,6 +228,48 @@ class LambdaFunction(BaseModel):
         return zip_output.read()
 
 
+class EventSourceMapping(BaseModel):
+
+    def __init__(self, spec):
+        # required
+        self.function_name = spec['FunctionName']
+        self.event_source_arn = spec['EventSourceArn']
+        self.starting_position = spec['StartingPosition']
+
+        # optional
+        self.batch_size = spec.get('BatchSize', 100)
+        self.enabled = spec.get('Enabled', True)
+        self.starting_position_timestamp = spec.get('StartingPositionTimestamp', None)
+
+    @classmethod
+    def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        properties = cloudformation_json['Properties']
+        spec = {
+            'FunctionName': properties['FunctionName'],
+            'EventSourceArn': properties['EventSourceArn'],
+            'StartingPosition': properties['StartingPosition']
+        }
+        optional_properties = 'BatchSize Enabled StartingPositionTimestamp'.split()
+        for prop in optional_properties:
+            if prop in properties:
+                spec[prop] = properties[prop]
+        return EventSourceMapping(spec)
+
+
+class LambdaVersion(BaseModel):
+
+    def __init__(self, spec):
+        self.version = spec['Version']
+
+    @classmethod
+    def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        properties = cloudformation_json['Properties']
+        spec = {
+            'Version': properties.get('Version')
+        }
+        return LambdaVersion(spec)
+
+
 class LambdaBackend(BaseBackend):
 
     def __init__(self):
@@ -242,6 +293,10 @@ class LambdaBackend(BaseBackend):
         return self._functions.values()
 
 
+def do_validate_s3():
+    return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true']
+
+
 lambda_backends = {}
 for region in boto.awslambda.regions():
     lambda_backends[region.name] = LambdaBackend()

moto/cloudformation/parsing.py

@@ -7,7 +7,9 @@ import warnings
 
 from moto.autoscaling import models as autoscaling_models
 from moto.awslambda import models as lambda_models
+from moto.cloudwatch import models as cloudwatch_models
 from moto.datapipeline import models as datapipeline_models
+from moto.dynamodb import models as dynamodb_models
 from moto.ec2 import models as ec2_models
 from moto.ecs import models as ecs_models
 from moto.elb import models as elb_models
@@ -27,7 +29,10 @@ from boto.cloudformation.stack import Output
 MODEL_MAP = {
     "AWS::AutoScaling::AutoScalingGroup": autoscaling_models.FakeAutoScalingGroup,
     "AWS::AutoScaling::LaunchConfiguration": autoscaling_models.FakeLaunchConfiguration,
+    "AWS::DynamoDB::Table": dynamodb_models.Table,
+    "AWS::Lambda::EventSourceMapping": lambda_models.EventSourceMapping,
     "AWS::Lambda::Function": lambda_models.LambdaFunction,
+    "AWS::Lambda::Version": lambda_models.LambdaVersion,
     "AWS::EC2::EIP": ec2_models.ElasticAddress,
     "AWS::EC2::Instance": ec2_models.Instance,
     "AWS::EC2::InternetGateway": ec2_models.InternetGateway,
@@ -53,6 +58,7 @@ MODEL_MAP = {
     "AWS::IAM::InstanceProfile": iam_models.InstanceProfile,
     "AWS::IAM::Role": iam_models.Role,
     "AWS::KMS::Key": kms_models.Key,
+    "AWS::Logs::LogGroup": cloudwatch_models.LogGroup,
     "AWS::RDS::DBInstance": rds_models.Database,
     "AWS::RDS::DBSecurityGroup": rds_models.SecurityGroup,
     "AWS::RDS::DBSubnetGroup": rds_models.SubnetGroup,
@@ -133,7 +139,7 @@ def clean_json(resource_json, resources_map):
             try:
                 return resource.get_cfn_attribute(resource_json['Fn::GetAtt'][1])
             except NotImplementedError as n:
-                logger.warning(n.message.format(
+                logger.warning(str(n).format(
                     resource_json['Fn::GetAtt'][0]))
             except UnformattedGetAttTemplateException:
                 raise ValidationError(

moto/cloudwatch/models.py

@@ -111,6 +111,27 @@ class CloudWatchBackend(BaseBackend):
         return self.metric_data
 
 
+class LogGroup(BaseModel):
+
+    def __init__(self, spec):
+        # required
+        self.name = spec['LogGroupName']
+        # optional
+        self.tags = spec.get('Tags', [])
+
+    @classmethod
+    def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        properties = cloudformation_json['Properties']
+        spec = {
+            'LogGroupName': properties['LogGroupName']
+        }
+        optional_properties = 'Tags'.split()
+        for prop in optional_properties:
+            if prop in properties:
+                spec[prop] = properties[prop]
+        return LogGroup(spec)
+
+
 cloudwatch_backends = {}
 for region in boto.ec2.cloudwatch.regions():
     cloudwatch_backends[region.name] = CloudWatchBackend()

moto/dynamodb/models.py

@@ -137,6 +137,20 @@ class Table(BaseModel):
             }
         return results
 
+    @classmethod
+    def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        properties = cloudformation_json['Properties']
+        key_attr = [i['AttributeName'] for i in properties['KeySchema'] if i['KeyType'] == 'HASH'][0]
+        key_type = [i['AttributeType'] for i in properties['AttributeDefinitions'] if i['AttributeName'] == key_attr][0]
+        spec = {
+            'name': properties['TableName'],
+            'hash_key_attr': key_attr,
+            'hash_key_type': key_type
+        }
+        # TODO: optional properties still missing:
+        # range_key_attr, range_key_type, read_capacity, write_capacity
+        return Table(**spec)
+
     def __len__(self):
         count = 0
         for key, value in self.items.items():
@@ -245,6 +259,14 @@ class Table(BaseModel):
         except KeyError:
             return None
 
+    def get_cfn_attribute(self, attribute_name):
+        from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
+        if attribute_name == 'StreamArn':
+            region = 'us-east-1'
+            time = '2000-01-01T00:00:00.000'
+            return 'arn:aws:dynamodb:{0}:123456789012:table/{1}/stream/{2}'.format(region, self.name, time)
+        raise UnformattedGetAttTemplateException()
+
 
 class DynamoDBBackend(BaseBackend):
 

tests/test_cloudformation/test_cloudformation_stack_crud.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import os
 import json
 
 import boto
@@ -565,3 +566,80 @@ def test_describe_stack_events_shows_create_update_and_delete():
         assert False, "Too many stack events"
 
     list(stack_events_to_look_for).should.be.empty
+
+
+@mock_cloudformation_deprecated
+@mock_route53_deprecated
+def test_create_stack_lambda_and_dynamodb():
+    conn = boto.connect_cloudformation()
+    dummy_template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Description": "Stack Lambda Test 1",
+        "Parameters": {
+        },
+        "Resources": {
+            "func1": {
+                "Type" : "AWS::Lambda::Function",
+                "Properties" : {
+                    "Code": {
+                        "S3Bucket": "bucket_123",
+                        "S3Key": "key_123"
+                    },
+                    "FunctionName": "func1",
+                    "Handler": "handler.handler",
+                    "Role": "role1",
+                    "Runtime": "python2.7",
+                    "Description": "descr",
+                    "MemorySize": 12345,
+                }
+            },
+            "func1version": {
+                "Type": "AWS::Lambda::LambdaVersion",
+                "Properties" : {
+                    "Version": "v1.2.3"
+                }
+            },
+            "tab1": {
+                "Type" : "AWS::DynamoDB::Table",
+                "Properties" : {
+                    "TableName": "tab1",
+                    "KeySchema": [{
+                        "AttributeName": "attr1",
+                        "KeyType": "HASH"
+                    }],
+                    "AttributeDefinitions": [{
+                        "AttributeName": "attr1",
+                        "AttributeType": "string"
+                    }],
+                    "ProvisionedThroughput": {
+                        "ReadCapacityUnits": 10,
+                        "WriteCapacityUnits": 10
+                    }
+                }
+            },
+            "func1mapping": {
+                "Type": "AWS::Lambda::EventSourceMapping",
+                "Properties" : {
+                    "FunctionName": "v1.2.3",
+                    "EventSourceArn": "arn:aws:dynamodb:region:XXXXXX:table/tab1/stream/2000T00:00:00.000",
+                    "StartingPosition": "0",
+                    "BatchSize": 100,
+                    "Enabled": True
+                }
+            }
+        },
+    }
+    validate_s3_before = os.environ.get('VALIDATE_LAMBDA_S3', '')
+    try:
+        os.environ['VALIDATE_LAMBDA_S3'] = 'false'
+        conn.create_stack(
+            "test_stack_lambda_1",
+            template_body=json.dumps(dummy_template),
+            parameters={}.items()
+        )
+    finally:
+        os.environ['VALIDATE_LAMBDA_S3'] = validate_s3_before
+
+    stack = conn.describe_stacks()[0]
+    resources = stack.list_resources()
+    assert len(resources) == 4